Updated ReadMe's, ToDo's and ChangeLog.

 * 1.6-release

Signed-off-by: Toni Uhlig <[email protected]>

CHANGELOG.md

@@ -1,5 +1,24 @@
 # CHANGELOG
 
+#### nDPId 1.6 (Nov 2023)
+
+ - Added Event I/O abstraction layer (supporting only poll/epoll by now)
+ - Support for OSX and *BSD systems
+ - Added proper DLT_RAW dissection for IPv4 and IPv6
+ - Improved TCP timeout handling if FIN/RST seen which caused Midstream TCP flows when there shouldn't be any
+ - Fixed a crash if `nDPId -o value=''` was used
+ - Added OpenWrt packaging
+ - Added new flow event "analyse" used to give some statistical information about active flows
+ - Added new analyse event daemon which generates CSV files from such events
+ - Fixed a crash in nDPIsrvd if a collector closes a connection
+ - Support `nDPId` to send it's data to a UDP endpoint instead of a nDPIsrvd collector
+ - Added events and flow states documentation
+ - Added basic systemd support
+ - Fixed a bug in base64 encoding which could lead to invalid base64 strings
+ - Added some machine learning examples
+ - Fixed various smaller bugs
+ - Fixed nDPIsrvd bug which causes invalid JSON strings sent to Distributors
+
 #### nDPId 1.5 (Apr 2022)
 
  - Improved nDPId cross compilation

CMakeLists.txt

@@ -26,7 +26,7 @@ set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON)
 set(CPACK_DEBIAN_DEBUGINFO_PACKAGE ON)
 set(CPACK_STRIP_FILES ON)
 set(CPACK_PACKAGE_VERSION_MAJOR 1)
-set(CPACK_PACKAGE_VERSION_MINOR 5)
+set(CPACK_PACKAGE_VERSION_MINOR 6)
 set(CPACK_PACKAGE_VERSION_PATCH 0)
 # Note: CPACK_PACKAGING_INSTALL_PREFIX and CMAKE_INSTALL_PREFIX are *not* the same.
 # It is used only to ease environment file loading via systemd.

README.md

@@ -12,6 +12,7 @@
 # References
 
 [ntop Webinar 2022](https://www.ntop.org/webinar/ntop-webinar-on-dec-14th-community-meeting-and-future-plans/)
+[ntopconf 2023](https://www.ntop.org/ntopconf2023/)
 
 # Disclaimer
 
@@ -304,6 +305,8 @@ Format: `subopt` (unit, comment): description
  * `max-packets-per-flow-to-send` (N, safe): max. `packet-flow` events that will be generated for the first N packets of each flow
  * `max-packets-per-flow-to-process` (N, caution advised): max. packets that will be processed by `libnDPI`
  * `max-packets-per-flow-to-analyze` (N, safe): max. packets to analyze before sending an `analyse` event, requires `-A`
+ * `error-event-threshold-n` (N, safe): max. error events to sent until threshold time passed by
+ * `error-event-threshold-time` (N, safe): time after which the error event thresold will be reset
 
 # test
 

TODO.md

@@ -1,16 +1,19 @@
 # TODOs
 
-1.6:
-
- * event I/O abstraction layer (testing)
- * Apple/BSD port (testing)
-
 1.7:
 
  * let nDPIsrvd (collector) connect to other nDPIsrvd instances (as distributor)
  * nDPIsrvd GnuTLS support for TCP/IP distributor connections
+ * PF\_RING integration
+
+2.0.0:
+
+ * switch to semantic versioning for the greater good ;)
 
 no release plan:
 
+ * merge flow end/idle event into idle event (end is not really useful..)
+ * provide a shared library for C / C++ for distributor application developers
  * improve UDP/TCP timeout handling by reading netfilter conntrack timeouts from /proc (or just read conntrack table entries)
  * detect interface / timeout changes and apply them to nDPId
+ * switch to MIT or BSD License

examples/README.md

@@ -13,10 +13,12 @@ Used also by `tests/run_tests.sh` if available.
 
 A capture daemon suitable for low-resource devices.
 It saves flows that were guessed/undetected/risky/midstream to a PCAP file for manual analysis.
+Used also by `tests/run_tests.sh` if available.
 
 ## c-collectd
 
 A collecd-exec compatible middleware that gathers statistic values from nDPId.
+Used also by `tests/run_tests.sh` if available.
 
 ## c-notifyd
 

test/results/caches_cfg/teams.pcap.out

@@ -1,5 +1,5 @@
-00564{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.5.0","ndpi_version":"4.9.0-4365-b08c787f","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
-00627{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.5.0","ndpi_version":"4.9.0-4365-b08c787f","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
+00564{"daemon_event_id":1,"daemon_event_name":"init","thread_id":0,"packet_id":0,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.9.0-4365-b08c787f","max-flows-per-thread":2048,"max-idle-flows-per-thread":64,"reader-thread-count":1,"flow-scan-interval":10000000,"generic-max-idle-time":600000000,"icmp-max-idle-time":120000000,"udp-max-idle-time":180000000,"tcp-max-idle-time":7560000000,"max-packets-per-flow-to-send":5,"max-packets-per-flow-to-process":32,"max-packets-per-flow-to-analyse":32,"global_ts_usec":0}
+00627{"daemon_event_id":4,"daemon_event_name":"status","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.9.0-4365-b08c787f","packets-captured":1,"packets-processed":0,"total-skipped-flows":0,"total-l4-payload-len":0,"total-not-detected-flows":0,"total-guessed-flows":0,"total-detected-flows":0,"total-detection-updates":0,"total-updates":0,"current-active-flows":0,"total-active-flows":0,"total-idle-flows":0,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":2,"global_ts_usec":1587041672419153}
 00776{"flow_event_id":1,"flow_event_name":"new","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5}
 00897{"packet_event_id":2,"packet_event_name":"packet-flow","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_packet_id":1,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"pkt_datalink":1,"pkt_caplen":321,"pkt_type":2048,"pkt_l3_offset":14,"pkt_l4_offset":34,"pkt_len":321,"pkt_l4_len":287,"thread_ts_usec":1587041672419153,"pkt":"\/\/\/\/\/\/\/\/2A0X1lWrCABFAAEzES1AAEARZ+TAqAAB\/\/\/\/\/wBEAEMBHwAAAQEGABgr52AAAIAAAAAAAAAAAAAAAAAAAAAAANgNF9ZVqwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABjglNjNQEBNwIBAwwJVEwtU0cxMTZFPAlUTC1TRzExNkU9BwHYDRfWVav\/"}
 01004{"flow_event_id":7,"flow_event_name":"detected","thread_id":0,"packet_id":1,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":1,"flow_state":"info","flow_src_packets_processed":1,"flow_dst_packets_processed":0,"flow_first_seen":1587041672419153,"flow_src_last_pkt_time":1587041672419153,"flow_dst_last_pkt_time":1587041672419153,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":279,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":279,"flow_dst_max_l4_payload_len":0,"flow_src_tot_l4_payload_len":279,"flow_dst_tot_l4_payload_len":0,"midstream":0,"thread_ts_usec":1587041672419153,"l3_proto":"ip4","src_ip":"192.168.0.1","dst_ip":"255.255.255.255","src_port":68,"dst_port":67,"l4_proto":"udp","ndpi": {"confidence": {"6":"DPI"},"proto":"DHCP","proto_id":"18","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Acceptable","category_id":14,"category":"Network","hostname":"tl-sg116e","dhcp": {"fingerprint":"1,3","class_ident":"TL-SG116E"}}}
@@ -678,7 +678,7 @@
 00970{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":75,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041694221137,"flow_src_last_pkt_time":1587041694221137,"flow_dst_last_pkt_time":1587041694234511,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":58,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":58,"flow_dst_max_l4_payload_len":134,"flow_src_tot_l4_payload_len":58,"flow_dst_tot_l4_payload_len":134,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":60837,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network"}}
 01002{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":53,"flow_state":"finished","flow_src_packets_processed":19,"flow_dst_packets_processed":13,"flow_first_seen":1587041687436782,"flow_src_last_pkt_time":1587041687725655,"flow_dst_last_pkt_time":1587041687725568,"flow_idle_time":7580000000,"flow_src_min_l4_payload_len":0,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":1313,"flow_dst_max_l4_payload_len":1440,"flow_src_tot_l4_payload_len":2206,"flow_dst_tot_l4_payload_len":7143,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"104.40.187.151","src_port":60562,"dst_port":443,"l4_proto":"tcp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"3":"DPI (partial)"},"proto":"TLS.Skype_Teams","proto_id":"91.125","proto_by_ip":"Azure","proto_by_ip_id":276,"encrypted":1,"breed":"Acceptable","category_id":10,"category":"VoIP"}}
 00970{"flow_event_id":3,"flow_event_name":"idle","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","flow_id":39,"flow_state":"finished","flow_src_packets_processed":1,"flow_dst_packets_processed":1,"flow_first_seen":1587041685093044,"flow_src_last_pkt_time":1587041685093044,"flow_dst_last_pkt_time":1587041685127636,"flow_idle_time":200000000,"flow_src_min_l4_payload_len":53,"flow_dst_min_l4_payload_len":0,"flow_src_max_l4_payload_len":53,"flow_dst_max_l4_payload_len":174,"flow_src_tot_l4_payload_len":53,"flow_dst_tot_l4_payload_len":174,"midstream":0,"thread_ts_usec":1587041698021081,"l3_proto":"ip4","src_ip":"192.168.1.6","dst_ip":"192.168.1.1","src_port":50653,"dst_port":53,"l4_proto":"udp","flow_datalink":1,"flow_max_packets":5,"ndpi": {"confidence": {"6":"DPI"},"proto":"DNS.Teams","proto_id":"5.250","proto_by_ip":"Unknown","proto_by_ip_id":0,"encrypted":0,"breed":"Safe","category_id":14,"category":"Network"}}
-00649{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.5.0","ndpi_version":"4.9.0-4365-b08c787f","packets-captured":1540,"packets-processed":1498,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":64,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":681,"global_ts_usec":1587041698021081}
+00649{"daemon_event_id":3,"daemon_event_name":"shutdown","thread_id":0,"packet_id":1540,"source":"cfgs\/caches_cfg\/pcap\/teams.pcap","alias":"nDPId-test","version":"1.6.0","ndpi_version":"4.9.0-4365-b08c787f","packets-captured":1540,"packets-processed":1498,"total-skipped-flows":0,"total-l4-payload-len":587095,"total-not-detected-flows":1,"total-guessed-flows":2,"total-detected-flows":80,"total-detection-updates":64,"total-updates":0,"current-active-flows":0,"total-active-flows":83,"total-idle-flows":83,"total-compressions":0,"total-compression-diff":0,"current-compression-diff":0,"total-events-serialized":681,"global_ts_usec":1587041698021081}
 ~~~~~~~~~~~~~~~~~~~~ SUMMARY ~~~~~~~~~~~~~~~~~~~~
 ~~ packets captured/processed: 1540/1498
 ~~ skipped flows.............: 0