Merge branch 'develop' into CYB-208

flink-cyber/flink-stellar/pom.xml

@@ -77,11 +77,6 @@
             <groupId>org.adrianwalker</groupId>
             <artifactId>multiline-string</artifactId>
         </dependency>
-      <dependency>
-            <groupId>com.trendmicro</groupId>
-            <artifactId>tlsh</artifactId>
-            <version>3.7.1</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-math3</artifactId>
@@ -252,6 +247,12 @@
             <version>${global_hamcrest_version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter</artifactId>
+            <version>${jupiter.junit.version}</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>

flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/BitPairsTable.java

@@ -0,0 +1,47 @@
+package org.apache.metron.stellar.common.utils.hashing.tlsh;
+
+class BitPairsTable {
+
+    private static final int BIT_PAIRS_DIFF_TABLE_SIZE = 256;
+
+    private final int[][] table;
+
+    BitPairsTable() {
+        this.table = generateDefaultBitPairsDiffTable();
+    }
+
+    private static int[][] generateDefaultBitPairsDiffTable() {
+        int[][] result = new int[BIT_PAIRS_DIFF_TABLE_SIZE][BIT_PAIRS_DIFF_TABLE_SIZE];
+
+        for (int i = 0; i < BIT_PAIRS_DIFF_TABLE_SIZE; i++) {
+            for (int j = 0; j < BIT_PAIRS_DIFF_TABLE_SIZE; j++) {
+                int x = i;
+                int y = j;
+                int diff = 0;
+
+                for (int z = 0; z < 4; z++) {
+                    int d = Math.abs(x % 4 - y % 4);
+
+                    if (d == 3) {
+                        diff += d * 2;
+                    } else {
+                        diff += d;
+                    }
+
+                    if (z < 3) {
+                        x /= 4;
+                        y /= 4;
+                    }
+                }
+
+                result[i][j] = diff;
+            }
+        }
+
+        return result;
+    }
+
+    public int getValue(int row, int column) {
+        return table[row][column];
+    }
+}

flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/SlidingWindow.java

@@ -0,0 +1,39 @@
+package org.apache.metron.stellar.common.utils.hashing.tlsh;
+
+
+import java.util.function.IntUnaryOperator;
+import java.util.stream.IntStream;
+
+public class SlidingWindow {
+    public static final int DEFAULT_SIZE = 5;
+    private final byte[] window;
+    private int byteCount = 0;
+
+    SlidingWindow() {
+        this.window = new byte[DEFAULT_SIZE];
+    }
+
+    public void put(final byte value) {
+        int cursor = byteCount % window.length;
+        window[cursor] = value;
+        byteCount++;
+    }
+
+    public int[] getWindow() {
+        final int startPosition = (byteCount - 1) % window.length;
+        final IntUnaryOperator reverseIterate = i -> i == 0 ? window.length - 1 : i - 1;
+        final IntUnaryOperator mapper = i -> window[i] & 0xFF;
+        return IntStream.iterate(startPosition, reverseIterate)
+                .limit(window.length)
+                .map(mapper)
+                .toArray();
+    }
+
+    public int getByteCount() {
+        return byteCount;
+    }
+
+    public int getWindowSize() {
+        return window.length;
+    }
+}

flink-cyber/flink-stellar/src/main/java/org/apache/metron/stellar/common/utils/hashing/tlsh/TLSH.java

@@ -17,42 +17,87 @@
  */
 package org.apache.metron.stellar.common.utils.hashing.tlsh;
 
-import com.trendmicro.tlsh.BucketOption;
-import com.trendmicro.tlsh.ChecksumOption;
-import com.trendmicro.tlsh.Tlsh;
-import com.trendmicro.tlsh.TlshCreator;
+import java.nio.ByteBuffer;
 
-import java.util.Optional;
+import static org.apache.metron.stellar.common.utils.hashing.tlsh.TLSHUtil.swapNibble;
 
 /**
  * The abstraction around interacting with TLSH.
  */
 public class TLSH {
-  private TlshCreator creator;
-  public TLSH(BucketOption bucketOption, ChecksumOption checksumOption) {
-    creator = new TlshCreator(bucketOption, checksumOption);
-  }
 
-  public String apply(byte[] data, boolean force) {
-    try {
-      creator.update(data);
-      return creator.getHash(force).getEncoded();
-    } finally {
-      creator.reset();
+    /**
+     * The checksum bytes.
+     */
+    private final int[] checksum;
+    /**
+     * The buckets bytes.
+     */
+    private final int[] codes;
+    /**
+     * The encoded length value.
+     */
+    private final int lValue;
+    /**
+     * The q1 ratio.
+     */
+    private final int q1Ratio;
+    /**
+     * The q2 ratio.
+     */
+    private final int q2Ratio;
+
+
+    public TLSH(int[] checksum, int[] codes, int lValue, int q1, int q2) {
+        this.checksum = checksum;
+        this.codes = codes;
+        this.lValue = lValue;
+        this.q1Ratio = q1;
+        this.q2Ratio = q2;
+    }
+
+
+    public String getHash() {
+        return TLSHUtil.bytesToHex(getHexBytes());
+    }
+
+    public int[] getChecksum() {
+        return checksum;
+    }
+
+    public int[] getCodes() {
+        return codes;
     }
-  }
 
-  public static int distance(String hash1, String hash2, Optional<Boolean> includeLength) {
-    if (hash1 == null || hash2 == null) {
-      return -1;
+    public int getlValue() {
+        return lValue;
     }
 
-    if (hash1.equals(hash2)) {
-      return 0;
+    public int getQ1Ratio() {
+        return q1Ratio;
     }
 
-    Tlsh t1 = Tlsh.fromTlshStr(hash1);
-    Tlsh t2 = Tlsh.fromTlshStr(hash2);
-    return t1.totalDiff(t2, includeLength.orElse(false));
-  }
+    public int getQ2Ratio() {
+        return q2Ratio;
+    }
+
+    public byte[] getHexBytes() {
+        final ByteBuffer buf = ByteBuffer.allocate(checksum.length + 2 + codes.length);
+        for (final int c : checksum) {
+            buf.put((byte) swapNibble(c));
+        }
+        buf.put((byte) swapNibble(lValue));
+        buf.put((byte) (q1Ratio << 4 | q2Ratio));
+        for (int i = codes.length - 1; i >= 0; i--) {
+            buf.put((byte) codes[i]);
+        }
+        buf.flip();
+        if (buf.hasArray() && 0 == buf.arrayOffset()) {
+            return buf.array();
+        } else {
+            final byte[] hash = new byte[buf.remaining()];
+            buf.get(hash);
+            return hash;
+        }
+    }
 }