diff --git a/flink-cyber/caracal-generator/pom.xml b/flink-cyber/caracal-generator/pom.xml
index a40060e5d..f825b12e9 100644
--- a/flink-cyber/caracal-generator/pom.xml
+++ b/flink-cyber/caracal-generator/pom.xml
@@ -33,6 +33,13 @@
flink-connector-kafka
+
+ org.apache.flink
+ flink-clients
+ provided
+ ${flink.version}
+
+
com.hortonworks.smm
monitoring-interceptors
diff --git a/flink-cyber/caracal-parser/pom.xml b/flink-cyber/caracal-parser/pom.xml
index f8b9e6f7c..45f034690 100644
--- a/flink-cyber/caracal-parser/pom.xml
+++ b/flink-cyber/caracal-parser/pom.xml
@@ -126,8 +126,12 @@
org.apache.flink
- flink-cloudera-registry
- ${flink.version}
+ flink-connector-cloudera-registry
+
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
@@ -164,6 +168,11 @@
flink-orc
+
+ org.apache.orc
+ orc-core
+
+
org.apache.parquet
parquet-avro
diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java
index f38eaab00..144aafabd 100644
--- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java
+++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java
@@ -54,7 +54,7 @@ public long getTs() {
return message.getTs();
}
- public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName())
+ public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null)
.fields()
.name("message").type(Message.SCHEMA$).noDefault()
.name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault()
diff --git a/flink-cyber/flink-alert-scoring/pom.xml b/flink-cyber/flink-alert-scoring/pom.xml
index 91a09d67c..1b6a740e0 100644
--- a/flink-cyber/flink-alert-scoring/pom.xml
+++ b/flink-cyber/flink-alert-scoring/pom.xml
@@ -95,6 +95,11 @@
org.apache.flink
flink-statebackend-rocksdb
+
+ com.google.guava
+ guava
+ test
+
diff --git a/flink-cyber/flink-commands/json-commands/pom.xml b/flink-cyber/flink-commands/json-commands/pom.xml
index 520d7b2a9..73a303e65 100644
--- a/flink-cyber/flink-commands/json-commands/pom.xml
+++ b/flink-cyber/flink-commands/json-commands/pom.xml
@@ -36,7 +36,6 @@
junit
junit
- test
@@ -70,15 +69,14 @@
${log4j.version}
provided
-
- com.google.guava
- guava
- ${guava.version}
-
com.fasterxml.jackson.core
jackson-databind
- ${jackson.version}
+
+
+ com.google.guava
+ guava
+ test
diff --git a/flink-cyber/flink-commands/scoring-commands/pom.xml b/flink-cyber/flink-commands/scoring-commands/pom.xml
index 5c2d9a8cb..8db7fc7d7 100644
--- a/flink-cyber/flink-commands/scoring-commands/pom.xml
+++ b/flink-cyber/flink-commands/scoring-commands/pom.xml
@@ -60,6 +60,14 @@
${jackson.datatype.version}
compile
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
+ com.google.guava
+ guava
+
org.slf4j
@@ -90,6 +98,10 @@
avro
compile
+
+ org.apache.flink
+ flink-java
+
@@ -128,6 +140,12 @@
+
+
+ com.google.
+ com.cloudera.cyber.com.google.
+
+
com.cloudera.cyber.scoring.ScoringJobKafka
diff --git a/flink-cyber/flink-common/pom.xml b/flink-cyber/flink-common/pom.xml
index 52401b821..53f49a4a7 100644
--- a/flink-cyber/flink-common/pom.xml
+++ b/flink-cyber/flink-common/pom.xml
@@ -43,9 +43,22 @@
org.apache.flink
flink-connector-kafka
+
+ org.apache.kafka
+ kafka-clients
+
+
+ org.apache.flink
+ flink-connector-cloudera-registry
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
org.apache.flink
- flink-cloudera-registry
+ flink-connector-base
+ ${flink.version}
org.apache.avro
diff --git a/flink-cyber/flink-cyber-api/pom.xml b/flink-cyber/flink-cyber-api/pom.xml
index f2548139e..2d72699bc 100644
--- a/flink-cyber/flink-cyber-api/pom.xml
+++ b/flink-cyber/flink-cyber-api/pom.xml
@@ -32,6 +32,19 @@
+
+
+
+
+ org.springframework.boot
+ spring-boot-dependencies
+ ${spring-boot.version}
+ pom
+ import
+
+
+
+
org.projectlombok
@@ -55,10 +68,15 @@
flink-table-common
+
+
+ org.springframework
+ spring-core
+
+
org.apache.avro
avro
- ${avro.version}
diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java
index 44e5dc8c1..ea7e61e16 100644
--- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java
+++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java
@@ -18,6 +18,7 @@
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.NonNull;
+import org.apache.avro.LogicalTypes;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.avro.specific.SpecificRecord;
@@ -56,7 +57,7 @@ public class Message extends SpecificRecordBase implements SpecificRecord, Ident
public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName())
.fields()
.requiredString("id")
- .requiredLong("ts")
+ .name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault()
.name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault()
.requiredString("message")
.name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$)))
diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java
index 9bec6f190..8309f15ac 100644
--- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java
+++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java
@@ -5,8 +5,16 @@
public class AvroSchemas {
public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName) {
- return SchemaBuilder.record(recordName).namespace(namespace)
- .prop("ssb.rowtimeAttribute", "ts")
- .prop("ssb.watermarkExpression", "`ts` - INTERVAL '30' SECOND");
+ return createRecordBuilder(namespace, recordName, "ts");
+ }
+
+ public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName, String tsFieldName) {
+ SchemaBuilder.RecordBuilder recordBuilder = SchemaBuilder.record(recordName).namespace(namespace);
+ if (tsFieldName != null) {
+ recordBuilder
+ .prop("ssb.rowtimeAttribute", tsFieldName)
+ .prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName));
+ }
+ return recordBuilder;
}
}
diff --git a/flink-cyber/flink-dedupe/pom.xml b/flink-cyber/flink-dedupe/pom.xml
index 4309f769e..17afd33b6 100644
--- a/flink-cyber/flink-dedupe/pom.xml
+++ b/flink-cyber/flink-dedupe/pom.xml
@@ -45,7 +45,12 @@
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
+
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml
index d21ad7006..deda184cf 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml
@@ -90,6 +90,11 @@
tests
+
+ com.google.guava
+ guava
+
+
com.hortonworks.smm
monitoring-interceptors
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml
index 2fb92f1e9..26c2abf46 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml
@@ -156,6 +156,11 @@
${commons-validator.version}
+
+ com.google.guava
+ guava
+
+
org.assertj
assertj-core
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml
index 94d4fe0be..e2cff6357 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml
@@ -72,25 +72,19 @@
org.apache.flink
flink-connector-hbase-${flink.hbase.version}
- ${flink.version}
-
-
- org.glassfish
- javax.el
-
-
- org.apache.hbase
- hbase-client
-
-
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
compile
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
com.cloudera.cyber
flink-cyber-api
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml
index 2c989478e..02b05df64 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml
@@ -39,6 +39,14 @@
lombok
provided
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+ com.fasterxml.jackson.core
+ jackson-databind
+
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml
index 0e6eb89a8..12b1ebb80 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml
@@ -52,21 +52,19 @@
org.apache.flink
flink-connector-hbase-${flink.hbase.version}
- ${flink.version}
-
-
- org.glassfish
- javax.el
-
-
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
provided
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
com.cloudera.cyber
flink-cyber-api
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml
index 17b0965be..af287c78e 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml
@@ -50,10 +50,15 @@
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
provided
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
com.cloudera.cyber
flink-cyber-api
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml
index aff14970e..26f45cfaa 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml
@@ -51,10 +51,15 @@
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
provided
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
com.cloudera.cyber
flink-cyber-api
@@ -118,10 +123,14 @@
${global.httpclient.version}
compile
+
+ junit
+ junit
+
org.mock-server
- mockserver-netty
- 5.11.1
+ mockserver-netty-no-dependencies
+ RELEASE
test
@@ -196,8 +205,8 @@
maven-surefire-plugin
${maven-surefire-plugin.version}
-
- always
+ 1
+ false
diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml
index 87db355a9..dadba9cfb 100644
--- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml
+++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml
@@ -47,21 +47,19 @@
org.apache.flink
flink-connector-hbase-${flink.hbase.version}
- ${flink.version}
-
-
- org.glassfish
- javax.el
-
-
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
provided
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
+
com.cloudera.cyber
flink-cyber-api
diff --git a/flink-cyber/flink-hbase-common/pom.xml b/flink-cyber/flink-hbase-common/pom.xml
index 461776d8d..e8180bd16 100644
--- a/flink-cyber/flink-hbase-common/pom.xml
+++ b/flink-cyber/flink-hbase-common/pom.xml
@@ -31,13 +31,6 @@
org.apache.flink
flink-connector-hbase-${flink.hbase.version}
- ${flink.version}
-
-
- org.glassfish
- javax.el
-
-
org.apache.hbase
diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml
index e24480a13..734aa9ae4 100644
--- a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml
+++ b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml
@@ -56,22 +56,47 @@
${project.parent.version}
+
+ org.apache.flink
+ flink-clients
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-table-runtime
+ ${flink.version}
+ provided
+
+
+
+ org.apache.flink
+ flink-avro
+
org.apache.flink
flink-streaming-java
+
+
+ com.esotericsoftware.kryo
+ kryo
+
+
org.apache.flink
- flink-table-api-java-bridge
+ flink-table-api-java
provided
+ ${flink.version}
org.apache.flink
- flink-table-planner_${scala.binary.version}
+ flink-table-planner-loader
${flink.version}
provided
@@ -88,13 +113,22 @@
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
+
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
org.apache.flink
flink-json
+
+ com.google.guava
+ guava
+
@@ -323,15 +357,18 @@
- junit
- junit
+ org.junit.jupiter
+ junit-jupiter
test
+ ${jupiter.junit.version}
+
org.hamcrest
hamcrest
test
+
org.mockito
mockito-core
@@ -344,6 +381,10 @@
2.2.0
test
+
+ org.apache.flink
+ flink-table-api-java-bridge
+
@@ -354,7 +395,7 @@
org.apache.maven.plugins
maven-shade-plugin
- 3.4.1
+ ${maven-shade-plugin.version}
@@ -370,6 +411,7 @@
org.slf4j:*
log4j:*
org.apache.logging.log4j:*
+ org.apache.avro:avro
@@ -381,6 +423,7 @@
META-INF/*.SF
META-INF/*.DSA
META-INF/*.RSA
+ org/apache/avro/**
@@ -400,6 +443,12 @@
org.apache.maven.plugins
maven-compiler-plugin
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+ ${maven-surefire-plugin.version}
+
\ No newline at end of file
diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java
new file mode 100644
index 000000000..935c1c1bd
--- /dev/null
+++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java
@@ -0,0 +1,40 @@
+package com.cloudera.cyber.indexing.hive.tableapi.impl;
+
+import com.cloudera.cyber.indexing.hive.util.AvroSchemaUtil;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.flink.api.common.functions.MapFunction;
+import org.apache.flink.api.common.typeinfo.TypeInformation;
+import org.apache.flink.api.java.typeutils.ResultTypeQueryable;
+import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo;
+import org.apache.flink.types.Row;
+
+import java.util.Set;
+
+public class MapRowToAvro implements ResultTypeQueryable, MapFunction {
+ private final GenericRecordAvroTypeInfo producedType;
+ private final Schema schema;
+
+ public MapRowToAvro(String schemaString) {
+ this.schema = new Schema.Parser().parse(schemaString);
+ this.producedType = new GenericRecordAvroTypeInfo(schema);
+ }
+
+ @Override
+ public TypeInformation getProducedType() {
+ return producedType;
+ }
+
+ @Override
+ public GenericRecord map(Row row) throws Exception {
+ final GenericRecord record = new GenericData.Record(schema);
+ final Set fieldNames = row.getFieldNames(true);
+ if (fieldNames != null) {
+ for (String fieldName : fieldNames) {
+ AvroSchemaUtil.putRowIntoAvro(row, record, fieldName);
+ }
+ }
+ return record;
+ }
+}
diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java
index 6d24c9c04..208545317 100644
--- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java
+++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java
@@ -55,18 +55,7 @@ protected void executeInsert(StreamTableEnvironment tableEnv, Map stream = tableEnv.toDataStream(table).map(row -> {
- final Schema schema = new Schema.Parser().parse(schemaString);
- final GenericRecord record = new GenericData.Record(schema);
- final Set fieldNames = row.getFieldNames(true);
- if (fieldNames != null) {
- for (String fieldName : fieldNames) {
- AvroSchemaUtil.putRowIntoAvro(row, record, fieldName);
- }
- }
-
- return record;
- });
+ final DataStream stream = tableEnv.toDataStream(table).map(new MapRowToAvro(schemaString));
stream.sinkTo(kafkaSink);
System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", mappingDto.getTableName(),
insertSql);
diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java
index ad7a5b294..7ded82282 100644
--- a/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java
+++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java
@@ -13,13 +13,15 @@
package com.cloudera.cyber.indexing.hive;
import com.cloudera.cyber.MessageUtils;
-import org.junit.Assert;
-import org.junit.Test;
+import org.junit.jupiter.api.Test;
import java.text.SimpleDateFormat;
import java.time.Instant;
import java.util.Date;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
public class TimestampNormalizerTest {
private final SimpleDateFormat hiveDateFormatter = new SimpleDateFormat(HiveStreamingMessageWriter.HIVE_DATE_FORMAT);
@@ -31,24 +33,24 @@ public void testLongTimestamp() {
long currentMillis = MessageUtils.getCurrentTimestamp();
String expectedTimestamp = hiveDateFormatter.format(Date.from(Instant.ofEpochMilli(currentMillis)));
String fieldValue = Long.toString(currentMillis);
- Assert.assertEquals(expectedTimestamp, normalizer.apply(fieldValue));
+ assertEquals(expectedTimestamp, normalizer.apply(fieldValue));
}
@Test
public void testFirstFormattedTimestamp() {
String fieldValue = "2020-11-19 22:00:01.000000";
- Assert.assertEquals("2020-11-19 22:00:01.000", normalizer.apply(fieldValue));
+ assertEquals("2020-11-19 22:00:01.000", normalizer.apply(fieldValue));
}
@Test
public void testLastFormattedTimestamp() {
String fieldValue = "2020-01-15T23:05:33Z";
- Assert.assertEquals("2020-01-15 23:05:33.000", normalizer.apply(fieldValue));
+ assertEquals("2020-01-15 23:05:33.000", normalizer.apply(fieldValue));
}
- @Test(expected=IllegalStateException.class)
+ @Test
public void testTimestampWithoutMatch() {
String fieldValue = "not a timestamp";
- normalizer.apply(fieldValue);
+ assertThrows(IllegalStateException.class, () -> normalizer.apply(fieldValue));
}
}
diff --git a/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml b/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml
index 11251b1bb..88fe32a62 100644
--- a/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml
+++ b/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml
@@ -56,7 +56,6 @@
org.apache.avro
avro
- ${avro.version}
org.apache.flink
@@ -68,6 +67,11 @@
flink-orc
+
+ org.apache.orc
+ orc-core
+
+
org.apache.parquet
parquet-avro
@@ -94,7 +98,7 @@
org.apache.maven.plugins
maven-shade-plugin
- 3.0.0
+ ${maven-shade-plugin.version}
diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml b/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml
index b054654d3..74857e384 100644
--- a/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml
+++ b/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml
@@ -84,6 +84,11 @@
flink-connector-kafka
+
+ com.google.guava
+ guava
+
+
org.apache.flink
flink-test-utils
@@ -99,7 +104,7 @@
org.apache.maven.plugins
maven-shade-plugin
- 3.0.0
+ ${maven-shade-plugin.version}
@@ -129,6 +134,12 @@
+
+
+ com.google.
+ com.cloudera.cyber.com.google.
+
+
diff --git a/flink-cyber/flink-indexing/pom.xml b/flink-cyber/flink-indexing/pom.xml
index 8f3c54d15..d9ab934f4 100644
--- a/flink-cyber/flink-indexing/pom.xml
+++ b/flink-cyber/flink-indexing/pom.xml
@@ -27,7 +27,6 @@
flink-indexing-parquet
flink-indexing-solr
- flink-indexing-elastic
flink-indexing-search
flink-indexing-hive
diff --git a/flink-cyber/flink-profiler-java/pom.xml b/flink-cyber/flink-profiler-java/pom.xml
index 424da0b37..15f1fae45 100644
--- a/flink-cyber/flink-profiler-java/pom.xml
+++ b/flink-cyber/flink-profiler-java/pom.xml
@@ -36,17 +36,14 @@
org.apache.flink
flink-connector-hbase-${flink.hbase.version}
- ${flink.version}
-
-
- org.glassfish
- javax.el
-
-
- org.apache.hbase
- hbase-client
-
-
+
+
+ org.apache.flink
+ flink-connector-cloudera-registry
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
com.cloudera.cyber
diff --git a/flink-cyber/flink-profiler/pom.xml b/flink-cyber/flink-profiler/pom.xml
index e5bc41fed..173f44311 100644
--- a/flink-cyber/flink-profiler/pom.xml
+++ b/flink-cyber/flink-profiler/pom.xml
@@ -59,6 +59,10 @@
${assertj.version}
test
+
+ com.google.guava
+ guava
+
diff --git a/flink-cyber/flink-sessions/pom.xml b/flink-cyber/flink-sessions/pom.xml
index 36cf88de1..5e2fc78ff 100644
--- a/flink-cyber/flink-sessions/pom.xml
+++ b/flink-cyber/flink-sessions/pom.xml
@@ -35,7 +35,12 @@
org.apache.flink
- flink-cloudera-registry
+ flink-connector-cloudera-registry
+
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
diff --git a/flink-cyber/flink-stellar/pom.xml b/flink-cyber/flink-stellar/pom.xml
index b153e0a34..4ed271838 100644
--- a/flink-cyber/flink-stellar/pom.xml
+++ b/flink-cyber/flink-stellar/pom.xml
@@ -35,7 +35,7 @@
UTF-8
1.10
${guava.version}
- 4.3.0.${cdh.version}
+ 5.4.0.${cdh.version}
0.66.19
2.7.4
1.1.1
diff --git a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
index 05a1f03c8..6e17ae48f 100644
--- a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
+++ b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java
@@ -45,20 +45,7 @@
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Append;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Durability;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Increment;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Row;
-import org.apache.hadoop.hbase.client.RowMutations;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
import org.apache.hadoop.hbase.filter.CompareFilter;
@@ -159,6 +146,11 @@ public TableDescriptor getDescriptor() throws IOException {
.build();
}
+ @Override
+ public RegionLocator getRegionLocator() throws IOException {
+ return null;
+ }
+
@Override
public boolean exists(Get get) throws IOException {
if (get.getFamilyMap() == null || get.getFamilyMap().size() == 0) {
@@ -598,7 +590,7 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, Compa
}
@Override
- public void mutateRow(RowMutations rowMutations) throws IOException {
+ public Result mutateRow(RowMutations rowMutations) throws IOException {
throw new UnsupportedOperationException();
}
diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml b/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml
index a14a5e157..353513de2 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml
+++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml
@@ -168,6 +168,11 @@
+
+ io.swagger
+ swagger-annotations
+ ${swagger-annotations.version}
+
diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml b/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml
index 8da4ed140..3d8a39291 100644
--- a/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml
+++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml
@@ -27,7 +27,6 @@
${antlr.version}
0.1.9
- 3.2.0
@@ -194,7 +193,7 @@
org.apache.maven.plugins
maven-shade-plugin
- ${global_shade_version}
+ ${maven-shade-plugin.version}
true
diff --git a/flink-cyber/metron-parser-chain/pom.xml b/flink-cyber/metron-parser-chain/pom.xml
index 2a8a1fb35..dc9c22406 100644
--- a/flink-cyber/metron-parser-chain/pom.xml
+++ b/flink-cyber/metron-parser-chain/pom.xml
@@ -41,7 +41,6 @@
UTF-8
- 3.2.1
3.8.1
3.2.0
3.1.1
@@ -50,7 +49,6 @@
1.18.12
- 2.7.6
1.2.2
1.7.15
diff --git a/flink-cyber/parser-chains-flink/pom.xml b/flink-cyber/parser-chains-flink/pom.xml
index 366efe323..650b1be65 100644
--- a/flink-cyber/parser-chains-flink/pom.xml
+++ b/flink-cyber/parser-chains-flink/pom.xml
@@ -33,6 +33,7 @@
org.apache.flink
flink-streaming-java
+
org.apache.flink
flink-clients
@@ -51,6 +52,28 @@
hbase-client
${hbase.version}
compile
+
+
+ org.slf4j
+ jul-to-slf4j
+
+
+ org.slf4j
+ jcl-over-slf4j
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ org.slf4j
+ slf4j-reload4j
+
+
@@ -84,6 +107,13 @@
flink-parquet
+
+ com.cloudera.cyber
+ flink-logging
+ ${project.parent.version}
+ compile
+
+
org.apache.parquet
parquet-avro
@@ -98,6 +128,26 @@
it.unimi.dsi
fastutil
+
+ org.slf4j
+ jul-to-slf4j
+
+
+ org.slf4j
+ jcl-over-slf4j
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ ch.qos.logback
+ logback-classic
+
+
+ org.slf4j
+ slf4j-reload4j
+
@@ -106,6 +156,24 @@
parser-chains-core
${project.parent.version}
compile
+
+
+ org.slf4j
+ jul-to-slf4j
+
+
+ org.slf4j
+ jcl-over-slf4j
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
+ ch.qos.logback
+ logback-classic
+
+
@@ -120,38 +188,6 @@
multiline-string
-
-
- org.slf4j
- slf4j-api
- ${slf4j.version}
-
-
-
- org.apache.logging.log4j
- log4j-slf4j-impl
- ${log4j.version}
-
-
-
- org.apache.logging.log4j
- log4j-api
- ${log4j.version}
-
-
-
- org.apache.logging.log4j
- log4j-core
- ${log4j.version}
-
-
-
-
- org.apache.logging.log4j
- log4j-1.2-api
- ${log4j.version}
-
-
org.assertj
assertj-core
@@ -243,7 +279,7 @@
META-INF/*.SF
META-INF/*.DSA
META-INF/*.RSA
-
+
diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml
index 160c8b54f..a84ce30cd 100644
--- a/flink-cyber/pom.xml
+++ b/flink-cyber/pom.xml
@@ -55,20 +55,22 @@
UTF-8
1.8
- 1.15.1-csa1.9.0.1
+ csa1.12.0.0
+ 1.18.0-csa1.12.0.0
+ 1.0-${csa.version}
2.4
- 7.1.7.0-551
+ 7.1.9.0-387
1.18.22
1.18.16.0
21.0
- 1.10.0
- 2.5.0.${cdh.version}
+ 1.11.3
+ 3.4.1.${cdh.version}
2.12
- 2.5.0.${cdh.version}
+ 3.4.1.${cdh.version}
1.11.4
2.11.2
2.10.1
- 2.1.0.${cdh.version}
+ 2.3.0.${cdh.version}
1.11.1
1.7.36
2.17.2
@@ -76,21 +78,21 @@
1.6.11
0.1.2
1.15.0
- 3.2.4
+ 3.3.0
3.2.0
3.0.0-M5
3.0.0-M5
3.3.1
- 8.4.1.${cdh.version}
+ 8.11.2.${cdh.version}
2.13.1
3.1.1.${cdh.version}
- 2.2.3.${cdh.version}
+ 2.4.17.${cdh.version}
3.6.0
1.3.0-incubating
3.17.2
1.1.1
5.9.3
- 5.1.0.${cdh.version}
+ 5.1.1.${cdh.version}
6.0.0.${cdh.version}
3.4
4.9.3
@@ -112,6 +114,8 @@
1.10.0
0.1.9
1.14.2
+ 1.5.1.${cdh.version}
+ 1.6.0
21.3.9
4.12
5.7.0
@@ -161,10 +165,26 @@
org.apache.flink
- flink-connector-kafka
+ flink-conector-base
${flink.version}
+
+ org.apache.flink
+ flink-connector-kafka
+ 3.1-${csa.version}
+
+
+ org.apache.kafka
+ kafka-clients
+
+
+ com.fasterxml.jackson.core
+ jackson-core
+
+
+
+
org.apache.flink
flink-connector-hive_${scala.binary.version}
@@ -217,6 +237,12 @@
flink-orc
${flink.version}
provided
+
+
+ org.apache.orc
+ orc-core
+
+
@@ -228,10 +254,21 @@
org.apache.flink
- flink-connector-hbase
- ${flink.version}
+ flink-connector-hbase-${flink.hbase.version}
+ 3.0-${csa.version}
+
+
+ org.apache.hbase
+ hbase-client
+
+
+
+ org.apache.orc
+ orc-core
+ ${orc.version}
+
com.github.erosb
@@ -241,17 +278,29 @@
org.apache.flink
- flink-cloudera-registry
- ${flink.version}
+ flink-connector-cloudera-registry
+ 1.0-${csa.version}
org.apache.avro
avro
+
+ org.apache.avro
+ avro
+
+
+ com.hortonworks.registries
+ schema-registry-serdes
+
org.everit.json
org.everit.json.schema
+
+ org.slf4j
+ log4j-over-slf4j
+
@@ -302,12 +351,6 @@
${flink.version}
-
- org.apache.flink
- flink-connector-elasticsearch7
- ${flink.version}
-
-
org.apache.solr
solr-solrj
@@ -379,7 +422,14 @@
com.hortonworks.registries
schema-registry-serdes
- 0.9.1
+ 0.10.0.${cdh.version}
+ compile
+
+
+ org.slf4j
+ log4j-over-slf4j
+
+
@@ -417,6 +467,13 @@
${jackson.version}
+
+ com.fasterxml.jackson.core
+ jackson-core
+ compile
+ ${jackson.version}
+
+
commons-validator
commons-validator
@@ -660,12 +717,7 @@
org.apache.flink
flink-connector-kafka
- ${flink.version}
-
-
- org.apache.flink
- flink-cloudera-registry
- ${flink.version}
+ 3.1-${csa.version}