[CYB-216] upgrade cybersec toolkit for csa 1.14.0.0 and cdh 7.3.1

cloudera · Dec 25, 2024 · 5f32abc · 5f32abc
1 parent b973766
commit 5f32abc
Show file tree

Hide file tree

Showing 4 changed files with 63 additions and 33 deletions.
diff --git a/...aracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java b/...aracal-parser/src/main/java/com/cloudera/cyber/caracal/SplitBroadcastProcessFunction.java
@@ -76,7 +76,7 @@ public void processBroadcastElement(SplitConfig splitConfig, Context context, Co
         log.info(String.format("Adding splitter %s on thread %d ", splitConfig, Thread.currentThread().getId()));
         context.getBroadcastState(SplitJob.Descriptors.broadcastState).put(splitConfig.getTopic(), splitConfig);
         SplittingFlatMapFunction f = new SplittingFlatMapFunction(splitConfig, signKey);
-        f.open(null);
+        f.open((Configuration)null);
         splitters.put(splitConfig.getTopic(), f);
     }
 }
diff --git a/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java b/flink-cyber/flink-common/src/main/java/com/cloudera/cyber/generator/ThreatGenerator.java
@@ -21,6 +21,7 @@
 import java.io.IOException;
 import java.io.StringWriter;
 import java.io.Writer;
+import java.nio.charset.StandardCharsets;
 import java.time.LocalDateTime;
 import java.time.ZoneOffset;
 import java.util.List;
@@ -36,16 +37,13 @@ public ThreatGenerator() {
         cfg.setClassLoaderForTemplateLoading(Thread.currentThread().getContextClassLoader(), "");
         cfg.setCacheStorage(new freemarker.cache.MruCacheStorage(50, 50));
         cfg.setTemplateUpdateDelayMilliseconds(3600 * 24 * 1000);
-        try {
-            lines = IOUtils.readLines(getClass().getResourceAsStream("/threats/threatq.json"));
-            StringTemplateLoader stringLoader = new StringTemplateLoader();
-            IntStream.range(0, lines.size()).forEach(i -> {
-                stringLoader.putTemplate(String.valueOf(i), lines.get(i));
-            });
-            cfg.setTemplateLoader(stringLoader);
-        } catch (IOException ioException) {
-            ioException.printStackTrace();
-        }
+
+        lines = IOUtils.readLines(getClass().getResourceAsStream("/threats/threatq.json"), StandardCharsets.UTF_8);
+        StringTemplateLoader stringLoader = new StringTemplateLoader();
+        IntStream.range(0, lines.size()).forEach(i -> {
+            stringLoader.putTemplate(String.valueOf(i), lines.get(i));
+        });
+        cfg.setTemplateLoader(stringLoader);
     }
 
     private static final RandomGenerators utils = new RandomGenerators();

diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml
@@ -257,6 +257,18 @@
                     <groupId>org.slf4j</groupId>
                     <artifactId>slf4j-log4j12</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.atlas</groupId>
+                    <artifactId>atlas-intg</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.cloudera.logredactor</groupId>
+                    <artifactId>logredactor</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.cloudera</groupId>
+                    <artifactId>jwtprovider-api</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -274,6 +286,18 @@
                     <groupId>commons-cli</groupId>
                     <artifactId>commons-cli</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-serde</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-metastore</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-exec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>
@@ -290,6 +314,10 @@
                     <groupId>commons-cli</groupId>
                     <artifactId>commons-cli</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>com.cloudera</groupId>
+                    <artifactId>jwtprovider-api</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 
@@ -303,6 +331,22 @@
                     <groupId>org.glassfish</groupId>
                     <artifactId>javax.el</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-cli</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-common</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-metastore</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-exec</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
 

diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml
@@ -55,23 +55,23 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <java.version>1.8</java.version>
-        <csa.version>csa1.12.0.0</csa.version>
-        <flink.version>1.18.0-csa1.12.0.0</flink.version>
+        <csa.version>csa1.14.0.0</csa.version>
+        <flink.version>1.19.1-${csa.version}</flink.version>
         <cloudera.registry>1.0-${csa.version}</cloudera.registry>
         <flink.hbase.version>2.4</flink.hbase.version>
-        <cdh.version>7.1.9.0-387</cdh.version>
+        <cdh.version>7.3.1.0-197</cdh.version>
         <lombok.version>1.18.22</lombok.version>
         <lombok.plugin.version>1.18.16.0</lombok.plugin.version>
         <guava.version>21.0</guava.version>
-        <avro.version>1.11.3</avro.version>
+        <avro.version>1.11.4</avro.version>
         <log4j.kafka.version>3.4.1.${cdh.version}</log4j.kafka.version>
         <scala.binary.version>2.12</scala.binary.version>
         <kafka.version>3.4.1.${cdh.version}</kafka.version>
         <nifi.version>1.11.4</nifi.version>
         <jackson.version>2.11.2</jackson.version>
         <jackson.datatype.version>2.10.1</jackson.datatype.version>
         <smm.intercepter.version>2.3.0.${cdh.version}</smm.intercepter.version>
-        <flink.format.parquet.version>1.11.1</flink.format.parquet.version>
+        <flink.format.parquet.version>1.12.2</flink.format.parquet.version>
         <slf4j.version>1.7.36</slf4j.version>
         <log4j.version>2.17.2</log4j.version>
         <spring-boot.version>2.7.6</spring-boot.version>
@@ -107,7 +107,7 @@
         <graalvm.version>21.3.10</graalvm.version>
         <junit.version>4.12</junit.version>
         <junit5.version>5.7.0</junit5.version>
-        <orc.version>1.5.1.${cdh.version}</orc.version>
+        <orc.version>1.8.3.${cdh.version}</orc.version>
         <swagger-annotations.version>1.6.0</swagger-annotations.version>
 
         <!--Commons-->
@@ -119,7 +119,7 @@
 
         <!--Maven-->
         <maven.build.timestamp.format>yyMMddHHmm</maven.build.timestamp.format>
-        <maven-shade-plugin.version>3.3.0</maven-shade-plugin.version>
+        <maven-shade-plugin.version>3.4.1</maven-shade-plugin.version>
         <maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
         <maven-source-plugin.version>3.3.1</maven-source-plugin.version>
         <maven-javadoc-plugin.version>3.6.3</maven-javadoc-plugin.version>
@@ -181,7 +181,7 @@
             <dependency>
                 <groupId>org.apache.flink</groupId>
                 <artifactId>flink-connector-kafka</artifactId>
-                <version>3.1-${csa.version}</version>
+                <version>3.2-${csa.version}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.apache.kafka</groupId>
@@ -290,18 +290,6 @@
                 <artifactId>flink-connector-cloudera-registry</artifactId>
                 <version>1.0-${csa.version}</version>
                 <exclusions>
-                    <exclusion>
-                        <groupId>org.apache.avro</groupId>
-                        <artifactId>avro</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>org.apache.avro</groupId>
-                        <artifactId>avro</artifactId>
-                    </exclusion>
-                    <exclusion>
-                        <groupId>com.hortonworks.registries</groupId>
-                        <artifactId>schema-registry-serdes</artifactId>
-                    </exclusion>
                     <exclusion>
                         <groupId>org.everit.json</groupId>
                         <artifactId>org.everit.json.schema</artifactId>
@@ -316,7 +304,7 @@
             <dependency>
                 <groupId>com.hortonworks.smm</groupId>
                 <artifactId>monitoring-interceptors</artifactId>
-                <version>[${smm.intercepter.version},)</version>
+                <version>${smm.intercepter.version}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.slf4j</groupId>
@@ -738,7 +726,7 @@
                 <dependency>
                     <groupId>org.apache.flink</groupId>
                     <artifactId>flink-connector-kafka</artifactId>
-                    <version>3.1-${csa.version}</version>
+                    <version>3.2-${csa.version}</version>
                 </dependency>
             </dependencies>
         </profile>