smb: configurable max number of transactions per flow

Ticket: OISF#5753

doc/userguide/configuration/suricata-yaml.rst

@@ -1722,7 +1722,7 @@ incompatible with ``decode-mime``. If both are enabled,
 Maximum transactions
 ~~~~~~~~~~~~~~~~~~~~
 
-MQTT, FTP, PostgreSQL and NFS have each a `max-tx` parameter that can be customized.
+MQTT, FTP, PostgreSQL, SMB and NFS have each a `max-tx` parameter that can be customized.
 `max-tx` refers to the maximum number of live transactions for each flow.
 An app-layer event `protocol.too_many_transactions` is triggered when this value is reached.
 The point of this parameter is to find a balance between the completeness of analysis

rules/smb-events.rules

@@ -41,4 +41,6 @@ alert smb any any -> any any (msg:"SURICATA SMB max READ queue size exceeded"; f
 # checks 'app-layer.protocols.smb.max-read-queue-cnt` against out of order chunks
 alert smb any any -> any any (msg:"SURICATA SMB max READ queue cnt exceeded"; flow:to_client; app-layer-event:smb.read_queue_cnt_exceeded; classtype:protocol-command-decode; sid:2225017; rev:1;)
 
-# next sid 2225018
+alert smb any any -> any any (msg:"SURICATA SMB too many transactions"; app-layer-event:smb.too_many_transactions; classtype:protocol-command-decode; sid:2225018; rev:1;)
+
+# next sid 2225019

rust/src/smb/events.rs

@@ -48,6 +48,8 @@ pub enum SMBEvent {
     WriteQueueCntExceeded,
     /// Unusal NTLMSSP fields order
     UnusualNtlmsspOrder,
+    /// Too many live transactions in one flow
+    TooManyTransactions,
 }
 
 impl SMBTransaction {

rust/src/smb/smb.rs

@@ -83,6 +83,8 @@ pub static mut SMB_CFG_MAX_WRITE_QUEUE_CNT: u32 = 0;
 
 static mut ALPROTO_SMB: AppProto = ALPROTO_UNKNOWN;
 
+static mut SMB_MAX_TX: usize = 1024;
+
 pub static mut SURICATA_SMB_FILE_CONFIG: Option<&'static SuricataFileContext> = None;
 
 #[no_mangle]
@@ -714,6 +716,7 @@ pub struct SMBState<> {
 
     /// transactions list
     pub transactions: VecDeque<SMBTransaction>,
+    tx_index_completed: usize,
 
     /// tx counter for assigning incrementing id's to tx's
     tx_id: u64,
@@ -770,6 +773,7 @@ impl SMBState {
             check_post_gap_file_txs: false,
             post_gap_files_checked: false,
             transactions: VecDeque::new(),
+            tx_index_completed: 0,
             tx_id:0,
             dialect:0,
             dialect_vec: None,
@@ -789,6 +793,20 @@ impl SMBState {
         self.tx_id += 1;
         tx.id = self.tx_id;
         SCLogDebug!("TX {} created", tx.id);
+        if self.transactions.len() > unsafe { SMB_MAX_TX } {
+            let mut index = self.tx_index_completed;
+            for tx_old in &mut self.transactions.range_mut(self.tx_index_completed..) {
+                index += 1;
+                if !tx_old.request_done || !tx_old.response_done {
+                    tx_old.request_done = true;
+                    tx_old.response_done = true;
+                    tx_old.set_event(SMBEvent::TooManyTransactions);
+                    break;
+                }
+            }
+            self.tx_index_completed = index;
+
+        }
         return tx;
     }
 
@@ -809,6 +827,7 @@ impl SMBState {
         if found {
             SCLogDebug!("freeing TX with ID {} TX.ID {} at index {} left: {} max id: {}",
                     tx_id, tx_id+1, index, self.transactions.len(), self.tx_id);
+            self.tx_index_completed = 0;
             self.transactions.remove(index);
         }
     }
@@ -2398,6 +2417,13 @@ pub unsafe extern "C" fn rs_smb_register_parser() {
                 Err(_) => { SCLogError!("Invalid max-read-queue-cnt value"); }
             }
         }
+        if let Some(val) = conf_get("app-layer.protocols.smb.max-tx") {
+            if let Ok(v) = val.parse::<usize>() {
+                SMB_MAX_TX = v;
+            } else {
+                SCLogError!("Invalid value for smb.max-tx");
+            }
+        }
     } else {
         SCLogDebug!("Protocol detector and parser disabled for SMB.");
     }

suricata.yaml.in

@@ -955,6 +955,8 @@ app-layer:
       enabled: yes
       detection-ports:
         dp: 139, 445
+      # Maximum number of live SMB transactions per flow
+      # max-tx: 1024
 
       # Stream reassembly size for SMB streams. By default track it completely.
       #stream-depth: 0