Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Decode runmode/v9 #11233

Closed
wants to merge 6 commits into from
Closed

Decode runmode/v9 #11233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
f8ec8e5
profiling: remove decode from packet profile line
victorjulien May 13, 2023
9156e91
runmodes: allow NULL decoder
victorjulien May 18, 2023
673a4be
pcap/file: prep for decoder merge
victorjulien May 13, 2023
a585c0a
capture: merge decode into capture
victorjulien May 14, 2023
b946759
dpdk: suppress scan-build warnings
victorjulien Jul 10, 2024
a29114e
threading: remove TM_FLAG_DECODE_TM flag
victorjulien May 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions src/decode-geneve.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ int DecodeGeneve(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t

if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GENEVE);
PacketEnqueueNoLock(&tv->decode_pq, tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
}

Expand Down Expand Up @@ -306,9 +306,9 @@ static int DecodeGeneveTest01(void)
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));

FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top == NULL);
FAIL_IF(dtv.decode_pq.top == NULL);

Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NOT(PacketIsUDP(tp));
FAIL_IF_NOT(tp->sp == 546);

Expand Down Expand Up @@ -348,9 +348,9 @@ static int DecodeGeneveTest02(void)
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));

FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top == NULL);
FAIL_IF(dtv.decode_pq.top == NULL);

Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NOT(PacketIsUDP(tp));
FAIL_IF_NOT(tp->sp == 53);

Expand Down Expand Up @@ -395,9 +395,9 @@ static int DecodeGeneveTest03(void)
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));

FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top == NULL);
FAIL_IF(dtv.decode_pq.top == NULL);

Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NOT(PacketIsUDP(tp));
FAIL_IF_NOT(tp->sp == 53);

Expand Down Expand Up @@ -439,7 +439,7 @@ static int DecodeGeneveTest04(void)
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));

FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top != NULL); /* Geneve packet should not have been processed */
FAIL_IF(dtv.decode_pq.top != NULL); /* Geneve packet should not have been processed */

DecodeGeneveConfigPorts(GENEVE_DEFAULT_PORT_S); /* Reset Geneve port list for future calls */
FlowShutdown();
Expand Down Expand Up @@ -479,7 +479,7 @@ static int DecodeGeneveTest05(void)
DecodeUDP(&tv, &dtv, p, raw_geneve, sizeof(raw_geneve));

FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top != NULL); /* Geneve packet should not have been processed */
FAIL_IF(dtv.decode_pq.top != NULL); /* Geneve packet should not have been processed */

FlowShutdown();
PacketFree(p);
Expand Down
14 changes: 7 additions & 7 deletions src/decode-gre.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
len - header_len, DECODE_TUNNEL_IPV4);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -221,7 +221,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
len - header_len, DECODE_TUNNEL_PPP);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -232,7 +232,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
len - header_len, DECODE_TUNNEL_IPV6);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -243,7 +243,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
len - header_len, DECODE_TUNNEL_VLAN);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -258,7 +258,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
GRE_FLAG_ISSET_SQ(greh) == 0 ? DECODE_TUNNEL_ERSPANI : DECODE_TUNNEL_ERSPANII);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -269,7 +269,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
len - header_len, DECODE_TUNNEL_ETHERNET);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand All @@ -279,7 +279,7 @@ int DecodeGRE(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *p
tv, dtv, p, pkt + header_len, len - header_len, DECODE_TUNNEL_ARP);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_GRE);
PacketEnqueueNoLock(&tv->decode_pq, tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
break;
}
Expand Down
10 changes: 5 additions & 5 deletions src/decode-ipv4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -540,7 +540,7 @@ int DecodeIPV4(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
if (unlikely(IPV4_GET_RAW_FRAGOFFSET(ip4h) > 0 || IPV4_GET_RAW_FLAG_MF(ip4h))) {
Packet *rp = Defrag(tv, dtv, p);
if (rp != NULL) {
PacketEnqueueNoLock(&tv->decode_pq, rp);
PacketEnqueueNoLock(&dtv->decode_pq, rp);
}
p->flags |= PKT_IS_FRAGMENT;
return TM_ECODE_OK;
Expand Down Expand Up @@ -591,7 +591,7 @@ int DecodeIPV4(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
Packet *tp = PacketTunnelPktSetup(tv, dtv, p, data, data_len, DECODE_TUNNEL_IPV6);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_IPV4);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
FlowSetupPacket(p);
break;
Expand Down Expand Up @@ -1307,7 +1307,7 @@ static int DecodeIPV4DefragTest01(void)
DecodeIPV4(&tv, &dtv, p, GET_PKT_DATA(p) + ETHERNET_HEADER_LEN,
GET_PKT_LEN(p) - ETHERNET_HEADER_LEN);
FAIL_IF(PacketIsTCP(p));
Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NULL(tp);
FAIL_IF(tp->recursion_level != p->recursion_level);
FAIL_IF_NOT(PacketIsIPv4(tp));
Expand Down Expand Up @@ -1404,7 +1404,7 @@ static int DecodeIPV4DefragTest02(void)
DecodeIPV4(&tv, &dtv, p, GET_PKT_DATA(p) + ETHERNET_HEADER_LEN,
GET_PKT_LEN(p) - ETHERNET_HEADER_LEN);
FAIL_IF(PacketIsTCP(p));
Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NULL(tp);
FAIL_IF(tp->recursion_level != p->recursion_level);
FAIL_IF_NOT(PacketIsIPv4(tp));
Expand Down Expand Up @@ -1502,7 +1502,7 @@ static int DecodeIPV4DefragTest03(void)
GET_PKT_LEN(p) - ETHERNET_HEADER_LEN);
FAIL_IF(PacketIsTCP(p));

Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NULL(tp);
FAIL_IF(!(tp->flags & PKT_WANTS_FLOW));
FAIL_IF(tp->flow_hash != p->flow_hash);
Expand Down
12 changes: 6 additions & 6 deletions src/decode-ipv6.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ static void DecodeIPv4inIPv6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, c
Packet *tp = PacketTunnelPktSetup(tv, dtv, p, pkt, plen, DECODE_TUNNEL_IPV4);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_IPV6);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
StatsIncr(tv, dtv->counter_ipv4inipv6);
return;
}
Expand All @@ -77,7 +77,7 @@ static int DecodeIP6inIP6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
Packet *tp = PacketTunnelPktSetup(tv, dtv, p, pkt, plen, DECODE_TUNNEL_IPV6);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_IPV6);
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
StatsIncr(tv, dtv->counter_ipv6inipv6);
}
} else {
Expand Down Expand Up @@ -642,7 +642,7 @@ int DecodeIPV6(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p, const uint8_t *
if (IPV6_EXTHDR_ISSET_FH(p)) {
Packet *rp = Defrag(tv, dtv, p);
if (rp != NULL) {
PacketEnqueueNoLock(&tv->decode_pq,rp);
PacketEnqueueNoLock(&dtv->decode_pq, rp);
}
}

Expand Down Expand Up @@ -797,7 +797,7 @@ static int DecodeIPV6FragTest01 (void)
goto end;
}

if (tv.decode_pq.len != 1) {
if (dtv.decode_pq.len != 1) {
printf("no reassembled packet: ");
goto end;
}
Expand All @@ -808,11 +808,11 @@ static int DecodeIPV6FragTest01 (void)
PacketRecycle(p2);
SCFree(p1);
SCFree(p2);
pkt = PacketDequeueNoLock(&tv.decode_pq);
pkt = PacketDequeueNoLock(&dtv.decode_pq);
while (pkt != NULL) {
PacketRecycle(pkt);
SCFree(pkt);
pkt = PacketDequeueNoLock(&tv.decode_pq);
pkt = PacketDequeueNoLock(&dtv.decode_pq);
}
DefragDestroy();
FlowShutdown();
Expand Down
2 changes: 1 addition & 1 deletion src/decode-teredo.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ int DecodeTeredo(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_TEREDO);
/* add the tp to the packet queue. */
PacketEnqueueNoLock(&tv->decode_pq,tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
StatsIncr(tv, dtv->counter_teredo);
return TM_ECODE_OK;
}
Expand Down
8 changes: 4 additions & 4 deletions src/decode-vxlan.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,7 +182,7 @@ int DecodeVXLAN(ThreadVars *tv, DecodeThreadVars *dtv, Packet *p,
len - (VXLAN_HEADER_LEN + ETHERNET_HEADER_LEN), decode_tunnel_proto);
if (tp != NULL) {
PKT_SET_SRC(tp, PKT_SRC_DECODER_VXLAN);
PacketEnqueueNoLock(&tv->decode_pq, tp);
PacketEnqueueNoLock(&dtv->decode_pq, tp);
}
}

Expand Down Expand Up @@ -219,9 +219,9 @@ static int DecodeVXLANtest01 (void)

DecodeUDP(&tv, &dtv, p, raw_vxlan, sizeof(raw_vxlan));
FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top == NULL);
FAIL_IF(dtv.decode_pq.top == NULL);

Packet *tp = PacketDequeueNoLock(&tv.decode_pq);
Packet *tp = PacketDequeueNoLock(&dtv.decode_pq);
FAIL_IF_NOT(PacketIsUDP(tp));
FAIL_IF_NOT(tp->sp == 53);

Expand Down Expand Up @@ -258,7 +258,7 @@ static int DecodeVXLANtest02 (void)

DecodeUDP(&tv, &dtv, p, raw_vxlan, sizeof(raw_vxlan));
FAIL_IF_NOT(PacketIsUDP(p));
FAIL_IF(tv.decode_pq.top != NULL);
FAIL_IF(dtv.decode_pq.top != NULL);

DecodeVXLANConfigPorts(VXLAN_DEFAULT_PORT_S); /* reset */
FlowShutdown();
Expand Down
3 changes: 3 additions & 0 deletions src/decode.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -944,6 +944,9 @@ static inline bool PacketIsARP(const Packet *p)
/** \brief Structure to hold thread specific data for all decode modules */
typedef struct DecodeThreadVars_
{
/** packet queue for decoders to store tunnel packets. */
PacketQueueNoLock decode_pq;

/** Specific context for udp protocol detection (here atm) */
AppLayerThreadCtx *app_tctx;

Expand Down
15 changes: 6 additions & 9 deletions src/runmode-af-packet.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -754,8 +754,8 @@ int RunModeIdsAFPAutoFp(void)
FatalError("Unable to init peers list.");
}

ret = RunModeSetLiveCaptureAutoFp(ParseAFPConfig, AFPConfigGeThreadsCount, "ReceiveAFP",
"DecodeAFP", thread_name_autofp, live_dev);
ret = RunModeSetLiveCaptureAutoFp(ParseAFPConfig, AFPConfigGeThreadsCount, "ReceiveAFP", NULL,
thread_name_autofp, live_dev);
if (ret != 0) {
FatalError("Unable to start runmode");
}
Expand Down Expand Up @@ -789,11 +789,8 @@ int RunModeIdsAFPSingle(void)
FatalError("Unable to init peers list.");
}

ret = RunModeSetLiveCaptureSingle(ParseAFPConfig,
AFPConfigGeThreadsCount,
"ReceiveAFP",
"DecodeAFP", thread_name_single,
live_dev);
ret = RunModeSetLiveCaptureSingle(ParseAFPConfig, AFPConfigGeThreadsCount, "ReceiveAFP", NULL,
thread_name_single, live_dev);
if (ret != 0) {
FatalError("Unable to start runmode");
}
Expand Down Expand Up @@ -830,8 +827,8 @@ int RunModeIdsAFPWorkers(void)
FatalError("Unable to init peers list.");
}

ret = RunModeSetLiveCaptureWorkers(ParseAFPConfig, AFPConfigGeThreadsCount, "ReceiveAFP",
"DecodeAFP", thread_name_workers, live_dev);
ret = RunModeSetLiveCaptureWorkers(ParseAFPConfig, AFPConfigGeThreadsCount, "ReceiveAFP", NULL,
thread_name_workers, live_dev);
if (ret != 0) {
FatalError("Unable to start runmode");
}
Expand Down
2 changes: 1 addition & 1 deletion src/runmode-dpdk.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1723,7 +1723,7 @@ int RunModeIdsDpdkWorkers(void)

InitEal();
ret = RunModeSetLiveCaptureWorkers(ParseDpdkConfigAndConfigureDevice, DPDKConfigGetThreadsCount,
"ReceiveDPDK", "DecodeDPDK", thread_name_workers, NULL);
"ReceiveDPDK", NULL, thread_name_workers, NULL);
if (ret != 0) {
FatalError("Unable to start runmode");
}
Expand Down
14 changes: 1 addition & 13 deletions src/runmode-pcap-file.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
/* Copyright (C) 2007-2010 Open Information Security Foundation
/* Copyright (C) 2007-2023 Open Information Security Foundation
*
* You can copy, redistribute or modify this Program under the terms of
* the GNU General Public License version 2 as published by the Free
Expand Down Expand Up @@ -80,12 +80,6 @@ int RunModeFilePcapSingle(void)
}
TmSlotSetFuncAppend(tv, tm_module, file);

tm_module = TmModuleGetByName("DecodePcapFile");
if (tm_module == NULL) {
FatalError("TmModuleGetByName DecodePcap failed");
}
TmSlotSetFuncAppend(tv, tm_module, NULL);

tm_module = TmModuleGetByName("FlowWorker");
if (tm_module == NULL) {
FatalError("TmModuleGetByName for FlowWorker failed");
Expand Down Expand Up @@ -175,12 +169,6 @@ int RunModeFilePcapAutoFp(void)
}
TmSlotSetFuncAppend(tv_receivepcap, tm_module, file);

tm_module = TmModuleGetByName("DecodePcapFile");
if (tm_module == NULL) {
FatalError("TmModuleGetByName DecodePcap failed");
}
TmSlotSetFuncAppend(tv_receivepcap, tm_module, NULL);

TmThreadSetCPU(tv_receivepcap, RECEIVE_CPU_SET);

if (TmThreadSpawn(tv_receivepcap) != TM_ECODE_OK) {
Expand Down
11 changes: 4 additions & 7 deletions src/runmode-pcap.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -234,11 +234,8 @@ int RunModeIdsPcapSingle(void)

(void)ConfGet("pcap.single-pcap-dev", &live_dev);

ret = RunModeSetLiveCaptureSingle(ParsePcapConfig,
PcapConfigGeThreadsCount,
"ReceivePcap",
"DecodePcap", thread_name_single,
live_dev);
ret = RunModeSetLiveCaptureSingle(ParsePcapConfig, PcapConfigGeThreadsCount, "ReceivePcap",
NULL, thread_name_single, live_dev);
if (ret != 0) {
FatalError("Runmode start failed");
}
Expand Down Expand Up @@ -274,7 +271,7 @@ int RunModeIdsPcapAutoFp(void)
(void) ConfGet("pcap.single-pcap-dev", &live_dev);

ret = RunModeSetLiveCaptureAutoFp(ParsePcapConfig, PcapConfigGeThreadsCount, "ReceivePcap",
"DecodePcap", thread_name_autofp, live_dev);
NULL, thread_name_autofp, live_dev);
if (ret != 0) {
FatalError("Runmode start failed");
}
Expand All @@ -301,7 +298,7 @@ int RunModeIdsPcapWorkers(void)
(void) ConfGet("pcap.single-pcap-dev", &live_dev);

ret = RunModeSetLiveCaptureWorkers(ParsePcapConfig, PcapConfigGeThreadsCount, "ReceivePcap",
"DecodePcap", thread_name_workers, live_dev);
NULL, thread_name_workers, live_dev);
if (ret != 0) {
FatalError("Unable to start runmode");
}
Expand Down
Loading
Loading