detect/dataset: delay set operation after signature full match

The set operation of dataset keyword was done even if signature
did not fully match, which is not the expected behavior.
We want dataset to behave like flowbits for instance.

This patch changes the behavior of the dataset keyword to do a
match and a post match for the set operation.
The postmatch retrieves the data, using the list identifier
associated to the buffer for this signature.

This avoids to store the buffer(s), when we do not have a
dedicated storage (per signature and per tx) that can own
and clean arbitrary buffers over multiple packets, in
the case the transaction spans over multiple packets
with different tx progresses for instance.

Thanks to delay_postmatch, the callback DetectDatasetTxMatch
should be called on the last packet taking the final match
decision. At this point, the inspection buffers are cached
and easy to get.
The most complex case is for multi-buffers, where we need
to know which buffers/local_ids matched. We use the ephemeral
storage (lives with one packet detection) provided by
DetectThreadCtxGetKeywordThreadCtx

For multi-buffers, we also return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF
to indicate that we want to know all the instances which match.

Ticket: #5576

src/detect-dataset.c

@@ -34,6 +34,7 @@
 #include "detect-engine.h"
 #include "detect-engine-mpm.h"
 #include "detect-engine-state.h"
+#include "detect-engine-content-inspection.h"
 
 #include "util-debug.h"
 #include "util-print.h"
@@ -47,66 +48,180 @@
 #define DETECT_DATASET_CMD_ISNOTSET 2
 #define DETECT_DATASET_CMD_ISSET    3
 
-int DetectDatasetMatch (ThreadVars *, DetectEngineThreadCtx *, Packet *,
-        const Signature *, const SigMatchCtx *);
+#define DMD_CAP_STEP 16
+typedef struct DetectDatasetMatchData_ {
+    uint32_t nb;
+    uint32_t *local_ids;
+} DetectDatasetMatchData;
+
 static int DetectDatasetSetup (DetectEngineCtx *, Signature *, const char *);
 void DetectDatasetFree (DetectEngineCtx *, void *);
 
+static int DetectDatasetTxMatch(DetectEngineThreadCtx *det_ctx, Flow *f, uint8_t flags, void *state,
+        void *txv, const Signature *s, const SigMatchCtx *ctx)
+{
+    const DetectDatasetData *sd = (DetectDatasetData *)ctx;
+    // This is only run for DETECT_SM_LIST_POSTMATCH
+    DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET);
+
+    // retrieve the app inspection engine associated to the list
+    DetectEngineAppInspectionEngine *a = s->app_inspect;
+    while (a != NULL) {
+        // also check alproto as http.uri as 2 engines : http1 and http2
+        if (a->sm_list == sd->list && a->alproto == f->alproto) {
+            if (a->v2.Callback == DetectEngineInspectBufferGeneric) {
+                // simple buffer, get data again
+                const InspectionBuffer *buffer =
+                        a->v2.GetData(det_ctx, a->v2.transforms, f, flags, txv, sd->list);
+                if (buffer != NULL && buffer->inspect != NULL) {
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        DatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            } else if (a->v2.Callback == DetectEngineInspectMultiBufferGeneric) {
+                DetectDatasetMatchData *dmd =
+                        (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                                det_ctx, sd->thread_ctx_id);
+                DEBUG_VALIDATE_BUG_ON(dmd == NULL);
+                uint32_t local_id = 0;
+                for (uint32_t i = 0; i < dmd->nb; i++) {
+                    local_id = dmd->local_ids[i];
+                    InspectionBuffer *buffer = a->v2.GetMultiData(
+                            det_ctx, a->v2.transforms, f, flags, txv, sd->list, local_id);
+                    DEBUG_VALIDATE_BUG_ON(buffer == NULL || buffer->inspect == NULL);
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        DatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            }
+            return 0;
+        }
+        a = a->next;
+    }
+    return 0;
+}
+
+static int DetectDatasetMatch(
+        DetectEngineThreadCtx *det_ctx, Packet *p, const Signature *s, const SigMatchCtx *ctx)
+{
+    const DetectDatasetData *sd = (DetectDatasetData *)ctx;
+    // This is only run for DETECT_SM_LIST_POSTMATCH
+    DEBUG_VALIDATE_BUG_ON(sd->cmd != DETECT_DATASET_CMD_SET && sd->cmd != DETECT_DATASET_CMD_UNSET);
+
+    // retrieve the pkt inspection engine associated to the list if any (ie if list is not a app
+    // inspection engine)
+    DetectEnginePktInspectionEngine *e = s->pkt_inspect;
+    while (e) {
+        if (e->sm_list == sd->list) {
+            if (e->v1.Callback == DetectEngineInspectPktBufferGeneric) {
+                const InspectionBuffer *buffer =
+                        e->v1.GetData(det_ctx, e->v1.transforms, p, sd->list);
+                // get simple data again and add it
+                if (buffer != NULL && buffer->inspect != NULL) {
+                    if (sd->cmd == DETECT_DATASET_CMD_SET) {
+                        DatasetAdd(sd->set, buffer->inspect, buffer->inspect_len);
+                    } else if (sd->cmd == DETECT_DATASET_CMD_UNSET) {
+                        DatasetRemove(sd->set, buffer->inspect, buffer->inspect_len);
+                    }
+                }
+            }
+            return 0;
+        }
+        e = e->next;
+    }
+    // return value is unused for postmatch functions
+    return 0;
+}
+
 void DetectDatasetRegister (void)
 {
     sigmatch_table[DETECT_DATASET].name = "dataset";
     sigmatch_table[DETECT_DATASET].desc = "match sticky buffer against datasets (experimental)";
     sigmatch_table[DETECT_DATASET].url = "/rules/dataset-keywords.html#dataset";
     sigmatch_table[DETECT_DATASET].Setup = DetectDatasetSetup;
     sigmatch_table[DETECT_DATASET].Free  = DetectDatasetFree;
+    // callbacks for postmatch
+    sigmatch_table[DETECT_DATASET].AppLayerTxMatch = DetectDatasetTxMatch;
+    sigmatch_table[DETECT_DATASET].Match = DetectDatasetMatch;
 }
 
 /*
     1 match
     0 no match
-    -1 can't match
  */
-int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
-    const DetectDatasetData *sd,
-    const uint8_t *data, const uint32_t data_len)
+uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd,
+        const uint8_t *data, const uint32_t data_len, uint32_t local_id)
 {
     if (data == NULL || data_len == 0)
-        return 0;
+        return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 
+    int r = DatasetLookup(sd->set, data, data_len);
+    SCLogDebug("r %d", r);
     switch (sd->cmd) {
         case DETECT_DATASET_CMD_ISSET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = DatasetLookup(sd->set, data, data_len);
-            SCLogDebug("r %d", r);
             if (r == 1)
-                return 1;
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
             break;
         }
         case DETECT_DATASET_CMD_ISNOTSET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = DatasetLookup(sd->set, data, data_len);
-            SCLogDebug("r %d", r);
             if (r < 1)
-                return 1;
+                return DETECT_ENGINE_INSPECT_SIG_MATCH;
             break;
         }
         case DETECT_DATASET_CMD_SET: {
-            //PrintRawDataFp(stdout, data, data_len);
-            int r = DatasetAdd(sd->set, data, data_len);
-            if (r == 1)
-                return 1;
-            break;
+            if (r == 1) {
+                /* Do not match if data is already in set */
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            // DatasetAdd will be performed postmatch if the rest of the sig completely matched
+            DetectDatasetMatchData *dmd =
+                    (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                            det_ctx, sd->thread_ctx_id);
+            if (dmd == NULL) {
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            if (dmd->nb % DMD_CAP_STEP == 0) {
+                void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP));
+                if (tmp == NULL) {
+                    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+                }
+                dmd->local_ids = tmp;
+            }
+            dmd->local_ids[dmd->nb] = local_id;
+            dmd->nb++;
+            return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF;
         }
         case DETECT_DATASET_CMD_UNSET: {
-            int r = DatasetRemove(sd->set, data, data_len);
-            if (r == 1)
-                return 1;
-            break;
+            if (r == 0) {
+                /* Do not match if data is not in set */
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            // DatasetRemove will be performed postmatch if the rest of the sig completely matched
+            DetectDatasetMatchData *dmd =
+                    (DetectDatasetMatchData *)DetectThreadCtxGetKeywordThreadCtx(
+                            det_ctx, sd->thread_ctx_id);
+            if (dmd == NULL) {
+                return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+            }
+            if (dmd->nb % DMD_CAP_STEP == 0) {
+                void *tmp = SCRealloc(dmd->local_ids, sizeof(uint32_t) * (dmd->nb + DMD_CAP_STEP));
+                if (tmp == NULL) {
+                    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
+                }
+                dmd->local_ids = tmp;
+            }
+            dmd->local_ids[dmd->nb] = local_id;
+            dmd->nb++;
+            return DETECT_ENGINE_INSPECT_SIG_MATCH_MORE_BUF;
         }
         default:
             DEBUG_VALIDATE_BUG_ON("unknown dataset command");
     }
-    return 0;
+    return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 }
 
 static int DetectDatasetParse(const char *str, char *cmd, int cmd_len, char *name, int name_len,
@@ -352,6 +467,24 @@ static int SetupSavePath(const DetectEngineCtx *de_ctx,
     return 0;
 }
 
+static void *DetectDatasetMatchDataThreadInit(void *data)
+{
+    DetectDatasetMatchData *scmd = SCCalloc(1, sizeof(DetectDatasetMatchData));
+    // make cocci happy
+    if (unlikely(scmd == NULL))
+        return NULL;
+    return scmd;
+}
+
+static void DetectDatasetMatchDataThreadFree(void *ctx)
+{
+    if (ctx) {
+        DetectDatasetMatchData *scmd = (DetectDatasetMatchData *)ctx;
+        SCFree(scmd->local_ids);
+        SCFree(scmd);
+    }
+}
+
 int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawstr)
 {
     DetectDatasetData *cd = NULL;
@@ -428,8 +561,30 @@ int DetectDatasetSetup (DetectEngineCtx *de_ctx, Signature *s, const char *rawst
     SCLogDebug("cmd %s, name %s",
         cmd_str, strlen(name) ? name : "(none)");
 
-    /* Okay so far so good, lets get this into a SigMatch
-     * and put it in the Signature. */
+    if (cmd == DETECT_DATASET_CMD_SET || cmd == DETECT_DATASET_CMD_UNSET) {
+        if (s->init_data->curbuf)
+            s->init_data->curbuf->delay_postmatch = true;
+        cd->thread_ctx_id = DetectRegisterThreadCtxFuncs(de_ctx, "dataset",
+                DetectDatasetMatchDataThreadInit, (void *)cd, DetectDatasetMatchDataThreadFree, 0);
+        if (cd->thread_ctx_id == -1)
+            goto error;
+
+        // for set operation, we need one match, and one postmatch
+        DetectDatasetData *scd = SCCalloc(1, sizeof(DetectDatasetData));
+        if (unlikely(scd == NULL))
+            goto error;
+
+        scd->set = set;
+        scd->cmd = cmd;
+        // remember the list used by match to retrieve the buffer in postmatch
+        scd->list = list;
+        scd->thread_ctx_id = cd->thread_ctx_id;
+        if (SigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)scd,
+                    DETECT_SM_LIST_POSTMATCH) == NULL) {
+            SCFree(scd);
+            goto error;
+        }
+    }
 
     if (SigMatchAppendSMToList(de_ctx, s, DETECT_DATASET, (SigMatchCtx *)cd, list) == NULL) {
         goto error;

src/detect-dataset.h

@@ -29,11 +29,13 @@
 typedef struct DetectDatasetData_ {
     Dataset *set;
     uint8_t cmd;
+    // for postmatch to retrieve the buffer(s)
+    int list;
+    int thread_ctx_id;
 } DetectDatasetData;
 
-int DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx,
-    const DetectDatasetData *sd,
-    const uint8_t *data, const uint32_t data_len);
+uint8_t DetectDatasetBufferMatch(DetectEngineThreadCtx *det_ctx, const DetectDatasetData *sd,
+        const uint8_t *data, const uint32_t data_len, uint32_t local_id);
 
 /* prototypes */
 void DetectDatasetRegister (void);

src/detect-engine-content-inspection.c

@@ -622,7 +622,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
 
         //PrintRawDataFp(stdout, buffer, buffer_len);
         const DetectDatasetData *sd = (const DetectDatasetData *) smd->ctx;
-        int r = DetectDatasetBufferMatch(det_ctx, sd, buffer, buffer_len); // TODO buffer offset?
+        int r = DetectDatasetBufferMatch(
+                det_ctx, sd, buffer, buffer_len, local_id); // TODO buffer offset?
         if (r == DETECT_ENGINE_INSPECT_SIG_MATCH) {
             goto match;
         }