Updated ReadMe's, ToDo's and ChangeLog.

 * 1.7-release

Signed-off-by: Toni Uhlig <[email protected]>

CHANGELOG.md

@@ -1,5 +1,24 @@
 # CHANGELOG
 
+#### nDPId 1.7 (Oct 2024)
+
+ - Read and parse configuration files for nDPId (+ libnDPI) and nDPIsrvd
+ - Added loading risk domains from a file (`-R`, thanks to @UnveilTech)
+ - Added Filebeat configuration file
+ - Improved hostname handling; will now always be part of `analyse`/`end`/`idle` events (if dissected)
+ - Improved Documentation (INSTALL / Schema)
+ - Added PF\_RING support
+ - Improved nDPIsrvd-analyse to write global stats to a CSV
+ - Added global (heap) memory stats for daemon status events (if enabled)
+ - Fixed IPv6 address/netmask retrieval on some systems
+ - Improved nDPIsrvd-collect; gauges and counters are now handled the right way
+ - Added nDPId Grafana dashboard
+ - Fixed `detection-update` event bug; was thrown even if nothing changed
+ - Fixed `not-detected` event spam if detection not completed (in some rare cases)
+ - Improved InfluxDB push daemon (severity parsing / gauge handling)
+ - Improved zLib compression
+ - Fixed nDPIsrvd-collectd missing escape character
+
 #### nDPId 1.6 (Nov 2023)
 
  - Added Event I/O abstraction layer (supporting only poll/epoll by now)

CMakeLists.txt

@@ -24,14 +24,14 @@ set(CPACK_PACKAGE_CONTACT "[email protected]")
 set(CPACK_DEBIAN_PACKAGE_NAME "nDPId")
 set(CPACK_DEBIAN_PACKAGE_SECTION "network")
 set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "nDPId is a set of daemons and tools to capture, process and classify network traffic.")
-set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Toni Uhlig")
+set(CPACK_DEBIAN_PACKAGE_MAINTAINER "Toni Uhlig <[email protected]>")
 set(CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_SOURCE_DIR}/packages/debian/preinst;${CMAKE_SOURCE_DIR}/packages/debian/prerm;${CMAKE_SOURCE_DIR}/packages/debian/postrm")
 set(CPACK_DEBIAN_PACKAGE_CONTROL_STRICT_PERMISSION TRUE)
 set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
 set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ON)
 set(CPACK_STRIP_FILES ON)
 set(CPACK_PACKAGE_VERSION_MAJOR 1)
-set(CPACK_PACKAGE_VERSION_MINOR 6)
+set(CPACK_PACKAGE_VERSION_MINOR 7)
 set(CPACK_PACKAGE_VERSION_PATCH 0)
 # Note: CPACK_PACKAGING_INSTALL_PREFIX and CMAKE_INSTALL_PREFIX are *not* the same.
 # It is used only to ease environment file loading via systemd.

TODO.md

@@ -1,10 +1,10 @@
 # TODOs
 
-1.7:
+1.8:
 
  * let nDPIsrvd (collector) connect to other nDPIsrvd instances (as distributor)
  * nDPIsrvd GnuTLS support for TCP/IP distributor connections
- * PF\_RING integration
+ * provide nDPId-exportd daemon which will only send captured packets to an nDPId instance running on a different machine
 
 2.0.0:
 

ndpid.conf.example

@@ -24,17 +24,20 @@
 # Set a name for this nDPId instance
 #alias       = myhostname
 
-# Process only internal initial connections
+# Process only internal initial connections (src->dst)
 #internal    = true
 
-# Process only external initial connections
+# Process only external initial connections (dst->src)
 #external    = true
 
 # Enable zLib compression of flow memory for long lasting flows
 compression = true
 
-# Enable "analyse" events, which can be used for machine learning
-analysis    = true
+# Enable "analyse" events, which can be used for machine learning.
+# The daemon will generate some statistical values for every single flow.
+# An "analyse" event is thrown after "max-packets-per-flow-to-analyse".
+# Please note that the daemon will require a lot more heap memory for every flow.
+#analysis    = true
 
 # Force poll() on systems that support epoll() as well
 #poll        = false

test/results/caches_cfg/teams.pcap.out

@@ -1,5 +1,5 @@
-00564{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.11.0-4834-92507c0","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
-00788{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.11.0-4834-92507c0","packets-captured":1,"packets-processed":0,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
+00564{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.7.0","ndpi_version":"4.11.0-4834-92507c0","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
+00788{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.7.0","ndpi_version":"4.11.0-4834-92507c0","packets-captured":1,"packets-processed":0,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
 00776{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5}
 00897{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"pkt_datalink":1,"pkt_caplen":321,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":321,"pkt_l4_len":287,"thread_ts_usec":1587041672419153,"pkt":"\/\/\/\/\/\/\/\/2A0X1lWrCABFAAEzES1AAEARZ+TAqAAB\/\/\/\/\/wBEAEMBHwAAAQEGABgr52AAAIAAAAAAAAAAAAAAAAAAAAAAANgNF9ZVqwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQEBNwIBAwwJVEwtU0cxMTZFPAlUTC1TRzExNkU9BwHYDRfWVav\/"}
 01004{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"DHCP","proto_id":"18","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network","hostname":"tl-sg116e","dhcp": {"fingerprint":"1,3","class_ident":"TL-SG116E"}}}
@@ -679,7 +679,7 @@
 01024{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":75,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041694221137,"flow_src_last_pkt_time":1587041694221137,"flow_dst_last_pkt_time":1587041694234511,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":58,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":58,"flow_dst_max_l4_payload_len":134,"flow_src_tot_l4_payload_len":58,"flow_dst_tot_l4_payload_len":134,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":60837,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network","hostname":"c-flightproxy-euno-01-teams.cloudapp.net"}}
 01039{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":53,"flow_state":"finished","flow_src_packets_processed":19,"flow_dst_packets_processed":13,"flow_first_seen":1587041687436782,"flow_src_last_pkt_time":1587041687725655,"flow_dst_last_pkt_time":1587041687725568,"flow_idle_time":7580000000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1313,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":2206,"flow_dst_tot_l4_payload_len":7143,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"104.40.187.151","src_port":60562,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"3":"DPI (partial)"},"proto":"TLS.Skype_Teams","proto_id":"91.125","proto_by_ip":"Azure","proto_by_ip_id":276,"encrypted":1,"breed":"Acceptable","category_id":10,"category":"VoIP","hostname":"api.microsoftstream.com"}}
 01019{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":39,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041685093044,"flow_src_last_pkt_time":1587041685093044,"flow_dst_last_pkt_time":1587041685127636,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":53,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":53,"flow_dst_max_l4_payload_len":174,"flow_src_tot_l4_payload_len":53,"flow_dst_tot_l4_payload_len":174,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":50653,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network","hostname":"api.flightproxy.teams.microsoft.com"}}
-00810{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.11.0-4834-92507c0","packets-captured":1540,"packets-processed":1498,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":65,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":682,"global_ts_usec":1587041698021081}
+00810{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.7.0","ndpi_version":"4.11.0-4834-92507c0","packets-captured":1540,"packets-processed":1498,"pfring_active":false,"pfring_recv":0,"pfring_drop":0,"pfring_shunt":0,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":65,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"global-alloc-count":0,"global-free-count":0,"global-alloc-bytes":0,"global-free-bytes":0,"total-events-serialized":682,"global_ts_usec":1587041698021081}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 1540/1498
 ~~ skipped flows.............: 0