diff --git a/src/Makefile.am b/src/Makefile.am index b8c28dcf6372..166555e8c0b8 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1289,7 +1289,8 @@ EXTRA_DIST = \ tests/detect-tls-version.c \ tests/detect-ipaddr.c \ tests/detect.c \ - tests/stream-tcp.c + tests/stream-tcp.c \ + tests/output-json-stats.c install-headers: mkdir -p $(DESTDIR)${includedir}/suricata diff --git a/src/detect-engine-iponly.c b/src/detect-engine-iponly.c index 63261ee716d5..3bd1eb04978e 100644 --- a/src/detect-engine-iponly.c +++ b/src/detect-engine-iponly.c @@ -81,16 +81,78 @@ static IPOnlyCIDRItem *IPOnlyCIDRItemNew(void) SCReturnPtr(item, "IPOnlyCIDRItem"); } -static uint8_t IPOnlyCIDRItemCompare(IPOnlyCIDRItem *head, - IPOnlyCIDRItem *item) +/** + * \brief Compares two list items + * + * \retval An integer less than, equal to, or greater than zero if lhs is + * considered to be respectively less than, equal to, or greater than + * rhs. + */ +static int IPOnlyCIDRItemCompareReal(const IPOnlyCIDRItem *lhs, const IPOnlyCIDRItem *rhs) { - uint8_t i = 0; - for (; i < head->netmask / 32 || i < 1; i++) { - if (item->ip[i] < head->ip[i]) - //if (*(uint8_t *)(item->ip + i) < *(uint8_t *)(head->ip + i)) - return 1; + if (lhs->netmask == rhs->netmask) { + uint8_t i = 0; + for (; i < lhs->netmask / 32 || i < 1; i++) { + if (lhs->ip[i] < rhs->ip[i]) + return -1; + if (lhs->ip[i] > rhs->ip[i]) + return 1; + } + return 0; } - return 0; + + return lhs->netmask < rhs->netmask ? -1 : 1; +} + +static int IPOnlyCIDRItemCompare(const void *lhsv, const void *rhsv) +{ + const IPOnlyCIDRItem *lhs = *(const IPOnlyCIDRItem **)lhsv; + const IPOnlyCIDRItem *rhs = *(const IPOnlyCIDRItem **)rhsv; + + return IPOnlyCIDRItemCompareReal(lhs, rhs); +} + +static void IPOnlyCIDRListQSort(IPOnlyCIDRItem **head) +{ + if (unlikely(head == NULL || *head == NULL)) + return; + + // First count the number of elements in the list + size_t len = 0; + IPOnlyCIDRItem *curr = *head; + + while (curr) { + curr = curr->next; + len++; + } + + // Place a pointer to the list item in an array for sorting + IPOnlyCIDRItem **tmp = SCMalloc(len * sizeof(IPOnlyCIDRItem *)); + + if (unlikely(tmp == NULL)) { + SCLogError("Failed to allocate enough memory to sort IP-only CIDR items."); + return; + } + + curr = *head; + for (size_t i = 0; i < len; i++) { + tmp[i] = curr; + curr = curr->next; + } + + // Perform the sort using the qsort algorithm + qsort(tmp, len, sizeof(IPOnlyCIDRItem *), IPOnlyCIDRItemCompare); + + // Update the links to the next element + *head = tmp[0]; + + for (size_t i = 0; i + 1 < len; i++) { + tmp[i]->next = tmp[i + 1]; + } + + tmp[len - 1]->next = NULL; + + SCFree(tmp); } //declaration for using it already @@ -348,11 +410,9 @@ static int IPOnlyCIDRItemSetup(IPOnlyCIDRItem **gh, char *s) return -1; } - /** * \brief This function insert a IPOnlyCIDRItem - * to a list of IPOnlyCIDRItems sorted by netmask - * ascending + * to a list of IPOnlyCIDRItems * \param head Pointer to the head of IPOnlyCIDRItems list * \param item Pointer to the item to insert in the list * @@ -361,37 +421,12 @@ static int IPOnlyCIDRItemSetup(IPOnlyCIDRItem **gh, char *s) static IPOnlyCIDRItem *IPOnlyCIDRItemInsertReal(IPOnlyCIDRItem *head, IPOnlyCIDRItem *item) { - IPOnlyCIDRItem *it, *prev = NULL; - if (item == NULL) return head; - /* Compare with the head */ - if (item->netmask < head->netmask || (item->netmask == head->netmask && IPOnlyCIDRItemCompare(head, item))) { - item->next = head; - return item; - } - - if (item->netmask == head->netmask && !IPOnlyCIDRItemCompare(head, item)) { - item->next = head->next; - head->next = item; - return head; - } - - for (prev = it = head; - it != NULL && it->netmask < item->netmask; - it = it->next) - prev = it; - - if (it == NULL) { - prev->next = item; - item->next = NULL; - } else { - item->next = it; - prev->next = item; - } - - return head; + /* Always insert item as head */ + item->next = head; + return item; } /** @@ -1108,6 +1143,9 @@ void IPOnlyPrepare(DetectEngineCtx *de_ctx) IPOnlyCIDRListPrint((de_ctx->io_ctx).ip_dst); */ + IPOnlyCIDRListQSort(&(de_ctx->io_ctx).ip_src); + IPOnlyCIDRListQSort(&(de_ctx->io_ctx).ip_dst); + IPOnlyCIDRItem *src, *dst; SCRadixNode *node = NULL; @@ -1725,64 +1763,124 @@ static int IPOnlyTestSig03 (void) static int IPOnlyTestSig04 (void) { int result = 1; - IPOnlyCIDRItem *head = NULL; - IPOnlyCIDRItem *new; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + // Test a linked list of size 0, 1, 2, ..., 5 + for (int size = 0; size < 6; size++) { + IPOnlyCIDRItem *new = NULL; - head = IPOnlyCIDRItemInsert(head, new); + if (size > 0) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 3; - new = IPOnlyCIDRItemNew(); - new->netmask= 11; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 1) { + new = IPOnlyCIDRItemNew(); + new->netmask = 11; - new = IPOnlyCIDRItemNew(); - new->netmask= 9; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 2) { + new = IPOnlyCIDRItemNew(); + new->netmask = 9; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 3) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 1; - new = IPOnlyCIDRItemNew(); - new->netmask= 10; + head = IPOnlyCIDRItemInsert(head, new); + } - head = IPOnlyCIDRItemInsert(head, new); + if (size > 4) { + new = IPOnlyCIDRItemNew(); + new->netmask = 10; + new->ip[0] = 2; - IPOnlyCIDRListPrint(head); - new = head; - if (new->netmask != 9) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 10) { - result = 0; - goto end; - } - new = new->next; - if (new->netmask != 11) { - result = 0; - goto end; + head = IPOnlyCIDRItemInsert(head, new); + } + + IPOnlyCIDRListPrint(head); + + IPOnlyCIDRListQSort(&head); + + if (size == 0) { + if (head != NULL) { + result = 0; + goto end; + } + } + + /** + * Validate the following list entries for each size + * 1 - 10 + * 2 - 10<3> 11 + * 3 - 9 10<3> 11 + * 4 - 9 10<1> 10<3> 11 + * 5 - 9 10<1> 10<2> 10<3> 11 + */ + new = head; + if (size >= 3) { + if (new->netmask != 9) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 4) { + if (new->netmask != 10 || new->ip[0] != 1) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 5) { + if (new->netmask != 10 || new->ip[0] != 2) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 1) { + if (new->netmask != 10 || new->ip[0] != 3) { + result = 0; + goto end; + } + new = new->next; + } + + if (size >= 2) { + if (new->netmask != 11) { + result = 0; + goto end; + } + new = new->next; + } + + if (new != NULL) { + result = 0; + goto end; + } + + IPOnlyCIDRListFree(head); + head = NULL; } end: - IPOnlyCIDRListFree(head); + if (head) { + IPOnlyCIDRListFree(head); + head = NULL; + } return result; } diff --git a/src/detect-filestore.c b/src/detect-filestore.c index c510544469aa..fa8492161e61 100644 --- a/src/detect-filestore.c +++ b/src/detect-filestore.c @@ -118,7 +118,8 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto switch (filestore->direction) { case FILESTORE_DIR_DEFAULT: rule_dir = 1; - break; + // will use both sides if scope is not default + // fallthrough case FILESTORE_DIR_BOTH: toserver_dir = 1; toclient_dir = 1; @@ -160,16 +161,28 @@ static int FilestorePostMatchWithOptions(Packet *p, Flow *f, const DetectFilesto AppLayerTxData *txd = AppLayerParserGetTxData(f->proto, f->alproto, txv); DEBUG_VALIDATE_BUG_ON(txd == NULL); if (txd != NULL) { - txd->file_flags |= FLOWFILE_STORE; + if (toclient_dir) { + txd->file_flags |= FLOWFILE_STORE_TC; + } + if (toserver_dir) { + txd->file_flags |= FLOWFILE_STORE_TS; + } } } } else if (this_flow) { /* set in flow and AppLayerStateData */ - f->file_flags |= FLOWFILE_STORE; - AppLayerStateData *sd = AppLayerParserGetStateData(f->proto, f->alproto, f->alstate); - if (sd != NULL) { - sd->file_flags |= FLOWFILE_STORE; + if (toclient_dir) { + f->file_flags |= FLOWFILE_STORE_TC; + if (sd != NULL) { + sd->file_flags |= FLOWFILE_STORE_TC; + } + } + if (toserver_dir) { + f->file_flags |= FLOWFILE_STORE_TS; + if (sd != NULL) { + sd->file_flags |= FLOWFILE_STORE_TS; + } } } else { FileStoreFileById(fc, file_id); diff --git a/src/flow.h b/src/flow.h index c7b5867ea896..418becfc2963 100644 --- a/src/flow.h +++ b/src/flow.h @@ -142,8 +142,9 @@ typedef struct AppLayerParserState_ AppLayerParserState; #define FLOWFILE_NO_SIZE_TS BIT_U16(10) #define FLOWFILE_NO_SIZE_TC BIT_U16(11) -/** store all files in the flow */ -#define FLOWFILE_STORE BIT_U16(12) +/** store files in the flow */ +#define FLOWFILE_STORE_TS BIT_U16(12) +#define FLOWFILE_STORE_TC BIT_U16(13) #define FLOWFILE_NONE_TS (FLOWFILE_NO_MAGIC_TS | \ FLOWFILE_NO_STORE_TS | \ diff --git a/src/output-json-stats.c b/src/output-json-stats.c index 718298e48592..33f98afa2dda 100644 --- a/src/output-json-stats.c +++ b/src/output-json-stats.c @@ -36,6 +36,7 @@ #include "util-print.h" #include "util-time.h" #include "util-unittest.h" +#include "util-validate.h" #include "util-debug.h" #include "output.h" @@ -265,20 +266,30 @@ json_t *StatsToJSON(const StatsTable *st, uint8_t flags) for (x = 0; x < st->ntstats; x++) { uint32_t offset = x * st->nstats; + // Stats for for this thread. + json_t *thread = json_object(); + if (unlikely(thread == NULL)) { + json_decref(js_stats); + json_decref(threads); + return NULL; + } + /* for each counter */ for (u = offset; u < (offset + st->nstats); u++) { if (st->tstats[u].name == NULL) continue; + // Seems this holds, but assert in debug builds. + DEBUG_VALIDATE_BUG_ON( + strcmp(st->tstats[offset].tm_name, st->tstats[u].tm_name) != 0); + json_t *js_type = NULL; const char *stat_name = st->tstats[u].short_name; if (st->tstats[u].short_name == NULL) { stat_name = st->tstats[u].name; js_type = threads; } else { - char str[256]; - snprintf(str, sizeof(str), "%s.%s", st->tstats[u].tm_name, st->tstats[u].name); - js_type = OutputStats2Json(threads, str); + js_type = OutputStats2Json(thread, st->tstats[u].name); } if (js_type != NULL) { @@ -292,6 +303,7 @@ json_t *StatsToJSON(const StatsTable *st, uint8_t flags) } } } + json_object_set_new(threads, st->tstats[offset].tm_name, thread); } json_object_set_new(js_stats, "threads", threads); } @@ -471,3 +483,7 @@ void JsonStatsLogRegister(void) { "eve-log.stats", OutputStatsLogInitSub, JsonStatsLogger, JsonStatsLogThreadInit, JsonStatsLogThreadDeinit, NULL); } + +#ifdef UNITTESTS +#include "tests/output-json-stats.c" +#endif diff --git a/src/output-json-stats.h b/src/output-json-stats.h index 9b96d5001298..b569e30b6429 100644 --- a/src/output-json-stats.h +++ b/src/output-json-stats.h @@ -35,4 +35,6 @@ TmEcode OutputEngineStatsReloadTime(json_t **jdata); TmEcode OutputEngineStatsRuleset(json_t **jdata); void JsonStatsLogRegister(void); +void OutputJsonStatsRegisterTests(void); + #endif /* __OUTPUT_JSON_COUNTERS_H__ */ diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index 1150bad89580..8ce0244146a3 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -114,6 +114,8 @@ #include "decode-vntag.h" #include "decode-vxlan.h" +#include "output-json-stats.h" + #ifdef OS_WIN32 #include "win32-syscall.h" #endif @@ -215,6 +217,7 @@ static void RegisterUnittests(void) #endif SCProtoNameRegisterTests(); UtilCIDRTests(); + OutputJsonStatsRegisterTests(); } #endif diff --git a/src/tests/output-json-stats.c b/src/tests/output-json-stats.c new file mode 100644 index 000000000000..ac1336eff898 --- /dev/null +++ b/src/tests/output-json-stats.c @@ -0,0 +1,70 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "../suricata-common.h" + +#include "../output-json-stats.h" + +#include "../util-unittest.h" + +static int OutputJsonStatsTest01(void) +{ + StatsRecord global_records[] = { { 0 }, { 0 } }; + StatsRecord thread_records[2]; + thread_records[0].name = "capture.kernel_packets"; + thread_records[0].short_name = "kernel_packets"; + thread_records[0].tm_name = "W#01-bond0.30"; + thread_records[0].value = 42; + thread_records[1].name = "capture.kernel_drops"; + thread_records[1].short_name = "kernel_drops"; + thread_records[1].tm_name = "W#01-bond0.30"; + thread_records[1].value = 4711; + + StatsTable table = { + .nstats = 2, + .stats = &global_records[0], + .ntstats = 1, + .tstats = &thread_records[0], + }; + + json_t *r = StatsToJSON(&table, JSON_STATS_TOTALS | JSON_STATS_THREADS); + if (!r) + return 0; + + // Remove variable content + json_object_del(r, "uptime"); + + char *serialized = json_dumps(r, 0); + + // Cheesy comparison + const char *expected = "{\"threads\": {\"W#01-bond0.30\": {\"capture\": {\"kernel_packets\": " + "42, \"kernel_drops\": 4711}}}}"; + + int cmp_result = strcmp(expected, serialized); + if (cmp_result != 0) + printf("unexpected result\nexpected=%s\ngot=%s\n", expected, serialized); + + free(serialized); + json_decref(r); + + return cmp_result == 0; +} + +void OutputJsonStatsRegisterTests(void) +{ + UtRegisterTest("OutputJsonStatsTest01", OutputJsonStatsTest01); +} diff --git a/src/util-file.c b/src/util-file.c index 3221d116870d..a1c30d89ecde 100644 --- a/src/util-file.c +++ b/src/util-file.c @@ -235,8 +235,11 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) uint16_t flags = 0; if (direction == STREAM_TOSERVER) { - if ((flow_file_flags & (FLOWFILE_NO_STORE_TS | FLOWFILE_STORE)) == FLOWFILE_NO_STORE_TS) { + if ((flow_file_flags & (FLOWFILE_NO_STORE_TS | FLOWFILE_STORE_TS)) == + FLOWFILE_NO_STORE_TS) { flags |= FILE_NOSTORE; + } else if (flow_file_flags & FLOWFILE_STORE_TS) { + flags |= FILE_STORE; } if (flow_file_flags & FLOWFILE_NO_MAGIC_TS) { @@ -255,8 +258,11 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) flags |= FILE_NOSHA256; } } else { - if ((flow_file_flags & (FLOWFILE_NO_STORE_TC | FLOWFILE_STORE)) == FLOWFILE_NO_STORE_TC) { + if ((flow_file_flags & (FLOWFILE_NO_STORE_TC | FLOWFILE_STORE_TC)) == + FLOWFILE_NO_STORE_TC) { flags |= FILE_NOSTORE; + } else if (flow_file_flags & FLOWFILE_STORE_TC) { + flags |= FILE_STORE; } if (flow_file_flags & FLOWFILE_NO_MAGIC_TC) { @@ -275,9 +281,6 @@ uint16_t FileFlowFlagsToFlags(const uint16_t flow_file_flags, uint8_t direction) flags |= FILE_NOSHA256; } } - if (flow_file_flags & FLOWFILE_STORE) { - flags |= FILE_STORE; - } DEBUG_VALIDATE_BUG_ON((flags & (FILE_STORE | FILE_NOSTORE)) == (FILE_STORE | FILE_NOSTORE)); SCLogDebug("direction %02x flags %02x", direction, flags);