diff --git a/flink-cyber/caracal-generator/pom.xml b/flink-cyber/caracal-generator/pom.xml index a40060e5d..f825b12e9 100644 --- a/flink-cyber/caracal-generator/pom.xml +++ b/flink-cyber/caracal-generator/pom.xml @@ -33,6 +33,13 @@ flink-connector-kafka + + org.apache.flink + flink-clients + provided + ${flink.version} + + com.hortonworks.smm monitoring-interceptors diff --git a/flink-cyber/caracal-parser/pom.xml b/flink-cyber/caracal-parser/pom.xml index f8b9e6f7c..45f034690 100644 --- a/flink-cyber/caracal-parser/pom.xml +++ b/flink-cyber/caracal-parser/pom.xml @@ -126,8 +126,12 @@ org.apache.flink - flink-cloudera-registry - ${flink.version} + flink-connector-cloudera-registry + + + + com.hortonworks.registries + schema-registry-serdes @@ -164,6 +168,11 @@ flink-orc + + org.apache.orc + orc-core + + org.apache.parquet parquet-avro diff --git a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java index f38eaab00..144aafabd 100644 --- a/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java +++ b/flink-cyber/flink-alert-scoring-api/src/main/java/com/cloudera/cyber/scoring/ScoredMessage.java @@ -54,7 +54,7 @@ public long getTs() { return message.getTs(); } - public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName()) + public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(ScoredMessage.class.getPackage().getName(), ScoredMessage.class.getName(), null) .fields() .name("message").type(Message.SCHEMA$).noDefault() .name("cyberScoresDetails").type(Schema.createArray(Scores.SCHEMA$)).noDefault() diff --git a/flink-cyber/flink-alert-scoring/pom.xml b/flink-cyber/flink-alert-scoring/pom.xml index 91a09d67c..1b6a740e0 100644 --- a/flink-cyber/flink-alert-scoring/pom.xml +++ b/flink-cyber/flink-alert-scoring/pom.xml @@ -95,6 +95,11 @@ org.apache.flink flink-statebackend-rocksdb + + com.google.guava + guava + test + diff --git a/flink-cyber/flink-commands/json-commands/pom.xml b/flink-cyber/flink-commands/json-commands/pom.xml index 520d7b2a9..73a303e65 100644 --- a/flink-cyber/flink-commands/json-commands/pom.xml +++ b/flink-cyber/flink-commands/json-commands/pom.xml @@ -36,7 +36,6 @@ junit junit - test @@ -70,15 +69,14 @@ ${log4j.version} provided - - com.google.guava - guava - ${guava.version} - com.fasterxml.jackson.core jackson-databind - ${jackson.version} + + + com.google.guava + guava + test diff --git a/flink-cyber/flink-commands/scoring-commands/pom.xml b/flink-cyber/flink-commands/scoring-commands/pom.xml index 5c2d9a8cb..8db7fc7d7 100644 --- a/flink-cyber/flink-commands/scoring-commands/pom.xml +++ b/flink-cyber/flink-commands/scoring-commands/pom.xml @@ -60,6 +60,14 @@ ${jackson.datatype.version} compile + + com.hortonworks.registries + schema-registry-serdes + + + com.google.guava + guava + org.slf4j @@ -90,6 +98,10 @@ avro compile + + org.apache.flink + flink-java + @@ -128,6 +140,12 @@ + + + com.google. + com.cloudera.cyber.com.google. + + com.cloudera.cyber.scoring.ScoringJobKafka diff --git a/flink-cyber/flink-common/pom.xml b/flink-cyber/flink-common/pom.xml index 52401b821..53f49a4a7 100644 --- a/flink-cyber/flink-common/pom.xml +++ b/flink-cyber/flink-common/pom.xml @@ -43,9 +43,22 @@ org.apache.flink flink-connector-kafka + + org.apache.kafka + kafka-clients + + + org.apache.flink + flink-connector-cloudera-registry + + + com.hortonworks.registries + schema-registry-serdes + org.apache.flink - flink-cloudera-registry + flink-connector-base + ${flink.version} org.apache.avro diff --git a/flink-cyber/flink-cyber-api/pom.xml b/flink-cyber/flink-cyber-api/pom.xml index f2548139e..2d72699bc 100644 --- a/flink-cyber/flink-cyber-api/pom.xml +++ b/flink-cyber/flink-cyber-api/pom.xml @@ -32,6 +32,19 @@ + + + + + org.springframework.boot + spring-boot-dependencies + ${spring-boot.version} + pom + import + + + + org.projectlombok @@ -55,10 +68,15 @@ flink-table-common + + + org.springframework + spring-core + + org.apache.avro avro - ${avro.version} diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java index 44e5dc8c1..ea7e61e16 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/Message.java @@ -18,6 +18,7 @@ import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; +import org.apache.avro.LogicalTypes; import org.apache.avro.Schema; import org.apache.avro.SchemaBuilder; import org.apache.avro.specific.SpecificRecord; @@ -56,7 +57,7 @@ public class Message extends SpecificRecordBase implements SpecificRecord, Ident public static final Schema SCHEMA$ = AvroSchemas.createRecordBuilder(Message.class.getPackage().getName(), Message.class.getName()) .fields() .requiredString("id") - .requiredLong("ts") + .name("ts").type(LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG))).noDefault() .name("originalSource").type(SignedSourceKey.SCHEMA$).noDefault() .requiredString("message") .name("threats").type().optional().type(SchemaBuilder.map().values(SchemaBuilder.array().items(ThreatIntelligence.SCHEMA$))) diff --git a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java index 9bec6f190..8309f15ac 100644 --- a/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java +++ b/flink-cyber/flink-cyber-api/src/main/java/com/cloudera/cyber/avro/AvroSchemas.java @@ -5,8 +5,16 @@ public class AvroSchemas { public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName) { - return SchemaBuilder.record(recordName).namespace(namespace) - .prop("ssb.rowtimeAttribute", "ts") - .prop("ssb.watermarkExpression", "`ts` - INTERVAL '30' SECOND"); + return createRecordBuilder(namespace, recordName, "ts"); + } + + public static SchemaBuilder.RecordBuilder createRecordBuilder(String namespace, String recordName, String tsFieldName) { + SchemaBuilder.RecordBuilder recordBuilder = SchemaBuilder.record(recordName).namespace(namespace); + if (tsFieldName != null) { + recordBuilder + .prop("ssb.rowtimeAttribute", tsFieldName) + .prop("ssb.watermarkExpression", String.format("`%s` - INTERVAL '30' SECOND", tsFieldName)); + } + return recordBuilder; } } diff --git a/flink-cyber/flink-dedupe/pom.xml b/flink-cyber/flink-dedupe/pom.xml index 4309f769e..17afd33b6 100644 --- a/flink-cyber/flink-dedupe/pom.xml +++ b/flink-cyber/flink-dedupe/pom.xml @@ -45,7 +45,12 @@ org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry + + + + com.hortonworks.registries + schema-registry-serdes diff --git a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml index d21ad7006..deda184cf 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-cidr/pom.xml @@ -90,6 +90,11 @@ tests + + com.google.guava + guava + + com.hortonworks.smm monitoring-interceptors diff --git a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml index 2fb92f1e9..26c2abf46 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-geocode/pom.xml @@ -156,6 +156,11 @@ ${commons-validator.version} + + com.google.guava + guava + + org.assertj assertj-core diff --git a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml index 94d4fe0be..e2cff6357 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-load/pom.xml @@ -72,25 +72,19 @@ org.apache.flink flink-connector-hbase-${flink.hbase.version} - ${flink.version} - - - org.glassfish - javax.el - - - org.apache.hbase - hbase-client - - org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry compile + + com.hortonworks.registries + schema-registry-serdes + + com.cloudera.cyber flink-cyber-api diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml index 2c989478e..02b05df64 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-common/pom.xml @@ -39,6 +39,14 @@ lombok provided + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-databind + diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml index 0e6eb89a8..12b1ebb80 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-hbase/pom.xml @@ -52,21 +52,19 @@ org.apache.flink flink-connector-hbase-${flink.hbase.version} - ${flink.version} - - - org.glassfish - javax.el - - org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry provided + + com.hortonworks.registries + schema-registry-serdes + + com.cloudera.cyber flink-cyber-api diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml index 17b0965be..af287c78e 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-raw/pom.xml @@ -50,10 +50,15 @@ org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry provided + + com.hortonworks.registries + schema-registry-serdes + + com.cloudera.cyber flink-cyber-api diff --git a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml index aff14970e..26f45cfaa 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-lookup-rest/pom.xml @@ -51,10 +51,15 @@ org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry provided + + com.hortonworks.registries + schema-registry-serdes + + com.cloudera.cyber flink-cyber-api @@ -118,10 +123,14 @@ ${global.httpclient.version} compile + + junit + junit + org.mock-server - mockserver-netty - 5.11.1 + mockserver-netty-no-dependencies + RELEASE test @@ -196,8 +205,8 @@ maven-surefire-plugin ${maven-surefire-plugin.version} - - always + 1 + false diff --git a/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml b/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml index 87db355a9..dadba9cfb 100644 --- a/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml +++ b/flink-cyber/flink-enrichment/flink-enrichment-threatq/pom.xml @@ -47,21 +47,19 @@ org.apache.flink flink-connector-hbase-${flink.hbase.version} - ${flink.version} - - - org.glassfish - javax.el - - org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry provided + + com.hortonworks.registries + schema-registry-serdes + + com.cloudera.cyber flink-cyber-api diff --git a/flink-cyber/flink-hbase-common/pom.xml b/flink-cyber/flink-hbase-common/pom.xml index 461776d8d..e8180bd16 100644 --- a/flink-cyber/flink-hbase-common/pom.xml +++ b/flink-cyber/flink-hbase-common/pom.xml @@ -31,13 +31,6 @@ org.apache.flink flink-connector-hbase-${flink.hbase.version} - ${flink.version} - - - org.glassfish - javax.el - - org.apache.hbase diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml index e24480a13..734aa9ae4 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml +++ b/flink-cyber/flink-indexing/flink-indexing-hive/pom.xml @@ -56,22 +56,47 @@ ${project.parent.version} + + org.apache.flink + flink-clients + ${flink.version} + provided + + + + org.apache.flink + flink-table-runtime + ${flink.version} + provided + + + + org.apache.flink + flink-avro + org.apache.flink flink-streaming-java + + + com.esotericsoftware.kryo + kryo + + org.apache.flink - flink-table-api-java-bridge + flink-table-api-java provided + ${flink.version} org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-planner-loader ${flink.version} provided @@ -88,13 +113,22 @@ org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry + + + + com.hortonworks.registries + schema-registry-serdes org.apache.flink flink-json + + com.google.guava + guava + @@ -323,15 +357,18 @@ - junit - junit + org.junit.jupiter + junit-jupiter test + ${jupiter.junit.version} + org.hamcrest hamcrest test + org.mockito mockito-core @@ -344,6 +381,10 @@ 2.2.0 test + + org.apache.flink + flink-table-api-java-bridge + @@ -354,7 +395,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.4.1 + ${maven-shade-plugin.version} @@ -370,6 +411,7 @@ org.slf4j:* log4j:* org.apache.logging.log4j:* + org.apache.avro:avro @@ -381,6 +423,7 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA + org/apache/avro/** @@ -400,6 +443,12 @@ org.apache.maven.plugins maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + \ No newline at end of file diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java new file mode 100644 index 000000000..935c1c1bd --- /dev/null +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/MapRowToAvro.java @@ -0,0 +1,40 @@ +package com.cloudera.cyber.indexing.hive.tableapi.impl; + +import com.cloudera.cyber.indexing.hive.util.AvroSchemaUtil; +import org.apache.avro.Schema; +import org.apache.avro.generic.GenericData; +import org.apache.avro.generic.GenericRecord; +import org.apache.flink.api.common.functions.MapFunction; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.api.java.typeutils.ResultTypeQueryable; +import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo; +import org.apache.flink.types.Row; + +import java.util.Set; + +public class MapRowToAvro implements ResultTypeQueryable, MapFunction { + private final GenericRecordAvroTypeInfo producedType; + private final Schema schema; + + public MapRowToAvro(String schemaString) { + this.schema = new Schema.Parser().parse(schemaString); + this.producedType = new GenericRecordAvroTypeInfo(schema); + } + + @Override + public TypeInformation getProducedType() { + return producedType; + } + + @Override + public GenericRecord map(Row row) throws Exception { + final GenericRecord record = new GenericData.Record(schema); + final Set fieldNames = row.getFieldNames(true); + if (fieldNames != null) { + for (String fieldName : fieldNames) { + AvroSchemaUtil.putRowIntoAvro(row, record, fieldName); + } + } + return record; + } +} diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java index 6d24c9c04..208545317 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/main/java/com/cloudera/cyber/indexing/hive/tableapi/impl/TableApiKafkaJob.java @@ -55,18 +55,7 @@ protected void executeInsert(StreamTableEnvironment tableEnv, Map stream = tableEnv.toDataStream(table).map(row -> { - final Schema schema = new Schema.Parser().parse(schemaString); - final GenericRecord record = new GenericData.Record(schema); - final Set fieldNames = row.getFieldNames(true); - if (fieldNames != null) { - for (String fieldName : fieldNames) { - AvroSchemaUtil.putRowIntoAvro(row, record, fieldName); - } - } - - return record; - }); + final DataStream stream = tableEnv.toDataStream(table).map(new MapRowToAvro(schemaString)); stream.sinkTo(kafkaSink); System.out.printf("Insert SQL added to the queue for the table: %s%nSQL: %s%n", mappingDto.getTableName(), insertSql); diff --git a/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java b/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java index ad7a5b294..7ded82282 100644 --- a/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java +++ b/flink-cyber/flink-indexing/flink-indexing-hive/src/test/java/com/cloudera/cyber/indexing/hive/TimestampNormalizerTest.java @@ -13,13 +13,15 @@ package com.cloudera.cyber.indexing.hive; import com.cloudera.cyber.MessageUtils; -import org.junit.Assert; -import org.junit.Test; +import org.junit.jupiter.api.Test; import java.text.SimpleDateFormat; import java.time.Instant; import java.util.Date; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + public class TimestampNormalizerTest { private final SimpleDateFormat hiveDateFormatter = new SimpleDateFormat(HiveStreamingMessageWriter.HIVE_DATE_FORMAT); @@ -31,24 +33,24 @@ public void testLongTimestamp() { long currentMillis = MessageUtils.getCurrentTimestamp(); String expectedTimestamp = hiveDateFormatter.format(Date.from(Instant.ofEpochMilli(currentMillis))); String fieldValue = Long.toString(currentMillis); - Assert.assertEquals(expectedTimestamp, normalizer.apply(fieldValue)); + assertEquals(expectedTimestamp, normalizer.apply(fieldValue)); } @Test public void testFirstFormattedTimestamp() { String fieldValue = "2020-11-19 22:00:01.000000"; - Assert.assertEquals("2020-11-19 22:00:01.000", normalizer.apply(fieldValue)); + assertEquals("2020-11-19 22:00:01.000", normalizer.apply(fieldValue)); } @Test public void testLastFormattedTimestamp() { String fieldValue = "2020-01-15T23:05:33Z"; - Assert.assertEquals("2020-01-15 23:05:33.000", normalizer.apply(fieldValue)); + assertEquals("2020-01-15 23:05:33.000", normalizer.apply(fieldValue)); } - @Test(expected=IllegalStateException.class) + @Test public void testTimestampWithoutMatch() { String fieldValue = "not a timestamp"; - normalizer.apply(fieldValue); + assertThrows(IllegalStateException.class, () -> normalizer.apply(fieldValue)); } } diff --git a/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml b/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml index 11251b1bb..88fe32a62 100644 --- a/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml +++ b/flink-cyber/flink-indexing/flink-indexing-parquet/pom.xml @@ -56,7 +56,6 @@ org.apache.avro avro - ${avro.version} org.apache.flink @@ -68,6 +67,11 @@ flink-orc + + org.apache.orc + orc-core + + org.apache.parquet parquet-avro @@ -94,7 +98,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.0.0 + ${maven-shade-plugin.version} diff --git a/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml b/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml index b054654d3..74857e384 100644 --- a/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml +++ b/flink-cyber/flink-indexing/flink-indexing-solr/pom.xml @@ -84,6 +84,11 @@ flink-connector-kafka + + com.google.guava + guava + + org.apache.flink flink-test-utils @@ -99,7 +104,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.0.0 + ${maven-shade-plugin.version} @@ -129,6 +134,12 @@ + + + com.google. + com.cloudera.cyber.com.google. + + diff --git a/flink-cyber/flink-indexing/pom.xml b/flink-cyber/flink-indexing/pom.xml index 8f3c54d15..d9ab934f4 100644 --- a/flink-cyber/flink-indexing/pom.xml +++ b/flink-cyber/flink-indexing/pom.xml @@ -27,7 +27,6 @@ flink-indexing-parquet flink-indexing-solr - flink-indexing-elastic flink-indexing-search flink-indexing-hive diff --git a/flink-cyber/flink-profiler-java/pom.xml b/flink-cyber/flink-profiler-java/pom.xml index 424da0b37..15f1fae45 100644 --- a/flink-cyber/flink-profiler-java/pom.xml +++ b/flink-cyber/flink-profiler-java/pom.xml @@ -36,17 +36,14 @@ org.apache.flink flink-connector-hbase-${flink.hbase.version} - ${flink.version} - - - org.glassfish - javax.el - - - org.apache.hbase - hbase-client - - + + + org.apache.flink + flink-connector-cloudera-registry + + + com.hortonworks.registries + schema-registry-serdes com.cloudera.cyber diff --git a/flink-cyber/flink-profiler/pom.xml b/flink-cyber/flink-profiler/pom.xml index e5bc41fed..173f44311 100644 --- a/flink-cyber/flink-profiler/pom.xml +++ b/flink-cyber/flink-profiler/pom.xml @@ -59,6 +59,10 @@ ${assertj.version} test + + com.google.guava + guava + diff --git a/flink-cyber/flink-sessions/pom.xml b/flink-cyber/flink-sessions/pom.xml index 36cf88de1..5e2fc78ff 100644 --- a/flink-cyber/flink-sessions/pom.xml +++ b/flink-cyber/flink-sessions/pom.xml @@ -35,7 +35,12 @@ org.apache.flink - flink-cloudera-registry + flink-connector-cloudera-registry + + + + com.hortonworks.registries + schema-registry-serdes diff --git a/flink-cyber/flink-stellar/pom.xml b/flink-cyber/flink-stellar/pom.xml index b153e0a34..4ed271838 100644 --- a/flink-cyber/flink-stellar/pom.xml +++ b/flink-cyber/flink-stellar/pom.xml @@ -35,7 +35,7 @@ UTF-8 1.10 ${guava.version} - 4.3.0.${cdh.version} + 5.4.0.${cdh.version} 0.66.19 2.7.4 1.1.1 diff --git a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java index 05a1f03c8..6e17ae48f 100644 --- a/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java +++ b/flink-cyber/metron-hbase-common/src/test/java/org/apache/metron/hbase/mock/MockHTable.java @@ -45,20 +45,7 @@ import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.Append; -import org.apache.hadoop.hbase.client.Delete; -import org.apache.hadoop.hbase.client.Durability; -import org.apache.hadoop.hbase.client.Get; -import org.apache.hadoop.hbase.client.Increment; -import org.apache.hadoop.hbase.client.Put; -import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Row; -import org.apache.hadoop.hbase.client.RowMutations; -import org.apache.hadoop.hbase.client.Scan; -import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.client.TableDescriptor; -import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.client.*; import org.apache.hadoop.hbase.client.coprocessor.Batch; import org.apache.hadoop.hbase.client.metrics.ScanMetrics; import org.apache.hadoop.hbase.filter.CompareFilter; @@ -159,6 +146,11 @@ public TableDescriptor getDescriptor() throws IOException { .build(); } + @Override + public RegionLocator getRegionLocator() throws IOException { + return null; + } + @Override public boolean exists(Get get) throws IOException { if (get.getFamilyMap() == null || get.getFamilyMap().size() == 0) { @@ -598,7 +590,7 @@ public boolean checkAndDelete(byte[] row, byte[] family, byte[] qualifier, Compa } @Override - public void mutateRow(RowMutations rowMutations) throws IOException { + public Result mutateRow(RowMutations rowMutations) throws IOException { throw new UnsupportedOperationException(); } diff --git a/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml b/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml index a14a5e157..353513de2 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml +++ b/flink-cyber/metron-parser-chain/parser-chains-config-service/pom.xml @@ -168,6 +168,11 @@ + + io.swagger + swagger-annotations + ${swagger-annotations.version} + diff --git a/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml b/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml index 8da4ed140..3d8a39291 100644 --- a/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml +++ b/flink-cyber/metron-parser-chain/parser-chains-parsers/pom.xml @@ -27,7 +27,6 @@ ${antlr.version} 0.1.9 - 3.2.0 @@ -194,7 +193,7 @@ org.apache.maven.plugins maven-shade-plugin - ${global_shade_version} + ${maven-shade-plugin.version} true diff --git a/flink-cyber/metron-parser-chain/pom.xml b/flink-cyber/metron-parser-chain/pom.xml index 2a8a1fb35..dc9c22406 100644 --- a/flink-cyber/metron-parser-chain/pom.xml +++ b/flink-cyber/metron-parser-chain/pom.xml @@ -41,7 +41,6 @@ UTF-8 - 3.2.1 3.8.1 3.2.0 3.1.1 @@ -50,7 +49,6 @@ 1.18.12 - 2.7.6 1.2.2 1.7.15 diff --git a/flink-cyber/parser-chains-flink/pom.xml b/flink-cyber/parser-chains-flink/pom.xml index 366efe323..650b1be65 100644 --- a/flink-cyber/parser-chains-flink/pom.xml +++ b/flink-cyber/parser-chains-flink/pom.xml @@ -33,6 +33,7 @@ org.apache.flink flink-streaming-java + org.apache.flink flink-clients @@ -51,6 +52,28 @@ hbase-client ${hbase.version} compile + + + org.slf4j + jul-to-slf4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + + org.slf4j + slf4j-reload4j + + @@ -84,6 +107,13 @@ flink-parquet + + com.cloudera.cyber + flink-logging + ${project.parent.version} + compile + + org.apache.parquet parquet-avro @@ -98,6 +128,26 @@ it.unimi.dsi fastutil + + org.slf4j + jul-to-slf4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + + org.slf4j + slf4j-reload4j + @@ -106,6 +156,24 @@ parser-chains-core ${project.parent.version} compile + + + org.slf4j + jul-to-slf4j + + + org.slf4j + jcl-over-slf4j + + + org.slf4j + log4j-over-slf4j + + + ch.qos.logback + logback-classic + + @@ -120,38 +188,6 @@ multiline-string - - - org.slf4j - slf4j-api - ${slf4j.version} - - - - org.apache.logging.log4j - log4j-slf4j-impl - ${log4j.version} - - - - org.apache.logging.log4j - log4j-api - ${log4j.version} - - - - org.apache.logging.log4j - log4j-core - ${log4j.version} - - - - - org.apache.logging.log4j - log4j-1.2-api - ${log4j.version} - - org.assertj assertj-core @@ -243,7 +279,7 @@ META-INF/*.SF META-INF/*.DSA META-INF/*.RSA - + diff --git a/flink-cyber/pom.xml b/flink-cyber/pom.xml index 160c8b54f..a84ce30cd 100644 --- a/flink-cyber/pom.xml +++ b/flink-cyber/pom.xml @@ -55,20 +55,22 @@ UTF-8 1.8 - 1.15.1-csa1.9.0.1 + csa1.12.0.0 + 1.18.0-csa1.12.0.0 + 1.0-${csa.version} 2.4 - 7.1.7.0-551 + 7.1.9.0-387 1.18.22 1.18.16.0 21.0 - 1.10.0 - 2.5.0.${cdh.version} + 1.11.3 + 3.4.1.${cdh.version} 2.12 - 2.5.0.${cdh.version} + 3.4.1.${cdh.version} 1.11.4 2.11.2 2.10.1 - 2.1.0.${cdh.version} + 2.3.0.${cdh.version} 1.11.1 1.7.36 2.17.2 @@ -76,21 +78,21 @@ 1.6.11 0.1.2 1.15.0 - 3.2.4 + 3.3.0 3.2.0 3.0.0-M5 3.0.0-M5 3.3.1 - 8.4.1.${cdh.version} + 8.11.2.${cdh.version} 2.13.1 3.1.1.${cdh.version} - 2.2.3.${cdh.version} + 2.4.17.${cdh.version} 3.6.0 1.3.0-incubating 3.17.2 1.1.1 5.9.3 - 5.1.0.${cdh.version} + 5.1.1.${cdh.version} 6.0.0.${cdh.version} 3.4 4.9.3 @@ -112,6 +114,8 @@ 1.10.0 0.1.9 1.14.2 + 1.5.1.${cdh.version} + 1.6.0 21.3.9 4.12 5.7.0 @@ -161,10 +165,26 @@ org.apache.flink - flink-connector-kafka + flink-conector-base ${flink.version} + + org.apache.flink + flink-connector-kafka + 3.1-${csa.version} + + + org.apache.kafka + kafka-clients + + + com.fasterxml.jackson.core + jackson-core + + + + org.apache.flink flink-connector-hive_${scala.binary.version} @@ -217,6 +237,12 @@ flink-orc ${flink.version} provided + + + org.apache.orc + orc-core + + @@ -228,10 +254,21 @@ org.apache.flink - flink-connector-hbase - ${flink.version} + flink-connector-hbase-${flink.hbase.version} + 3.0-${csa.version} + + + org.apache.hbase + hbase-client + + + + org.apache.orc + orc-core + ${orc.version} + com.github.erosb @@ -241,17 +278,29 @@ org.apache.flink - flink-cloudera-registry - ${flink.version} + flink-connector-cloudera-registry + 1.0-${csa.version} org.apache.avro avro + + org.apache.avro + avro + + + com.hortonworks.registries + schema-registry-serdes + org.everit.json org.everit.json.schema + + org.slf4j + log4j-over-slf4j + @@ -302,12 +351,6 @@ ${flink.version} - - org.apache.flink - flink-connector-elasticsearch7 - ${flink.version} - - org.apache.solr solr-solrj @@ -379,7 +422,14 @@ com.hortonworks.registries schema-registry-serdes - 0.9.1 + 0.10.0.${cdh.version} + compile + + + org.slf4j + log4j-over-slf4j + + @@ -417,6 +467,13 @@ ${jackson.version} + + com.fasterxml.jackson.core + jackson-core + compile + ${jackson.version} + + commons-validator commons-validator @@ -660,12 +717,7 @@ org.apache.flink flink-connector-kafka - ${flink.version} - - - org.apache.flink - flink-cloudera-registry - ${flink.version} + 3.1-${csa.version}