diff --git a/examples/plugins/c-json-filetype/Makefile.in b/examples/plugins/c-json-filetype/Makefile.in index 0d4ec381ee4a..4d85c8ee2d59 100644 --- a/examples/plugins/c-json-filetype/Makefile.in +++ b/examples/plugins/c-json-filetype/Makefile.in @@ -5,7 +5,7 @@ # But as this is an example in the Suricata source tree we'll look for # includes in the source tree. -CPPFLAGS += -I@top_srcdir@/src -DHAVE_CONFIG_H +CPPFLAGS += -I@top_srcdir@/src -I@top_srcdir@/rust/gen -I@top_srcdir@/rust/dist -DHAVE_CONFIG_H # Currently the Suricata logging system requires this to be even for # plugins. diff --git a/src/Makefile.am b/src/Makefile.am index 7d05751ec254..b0f841cfd0c2 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -317,6 +317,7 @@ noinst_HEADERS = \ feature.h \ flow-bit.h \ flow-bypass.h \ + flow-callbacks.h \ flow.h \ flow-hash.h \ flow-manager.h \ @@ -437,6 +438,8 @@ noinst_HEADERS = \ suricata-common.h \ suricata.h \ suricata-plugin.h \ + thread-callbacks.h \ + thread-storage.h \ threads-debug.h \ threads.h \ threads-profile.h \ @@ -875,6 +878,7 @@ libsuricata_c_a_SOURCES = \ feature.c \ flow-bit.c \ flow-bypass.c \ + flow-callbacks.c \ flow.c \ flow-hash.c \ flow-manager.c \ @@ -988,6 +992,8 @@ libsuricata_c_a_SOURCES = \ stream-tcp-sack.c \ stream-tcp-util.c \ suricata.c \ + thread-callbacks.c \ + thread-storage.c \ threads.c \ tm-modules.c \ tmqh-flow.c \ @@ -1168,6 +1174,12 @@ install-headers: for header in $(noinst_HEADERS); do \ $(INSTALL_DATA) $$header "$(DESTDIR)${includedir}/suricata"; \ done + if test -e ../rust/dist/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/dist/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi + if test -e ../rust/gen/rust-bindings.h; then \ + $(INSTALL_DATA) ../rust/gen/rust-bindings.h "$(DESTDIR)${includedir}/suricata"; \ + fi # Until we can remove autoconf.h from our headers, we need to to # provide this for library/plugin users. diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c index 37fbc98d8597..903cc158cf5f 100644 --- a/src/detect-engine-register.c +++ b/src/detect-engine-register.c @@ -439,7 +439,7 @@ void SigTableCleanup(void) } } -void SigTableSetup(void) +void SigTableInit(void) { if (sigmatch_table == NULL) { DETECT_TBLSIZE = DETECT_TBLSIZE_STATIC + DETECT_TBLSIZE_STEP; @@ -447,10 +447,12 @@ void SigTableSetup(void) if (sigmatch_table == NULL) { DETECT_TBLSIZE = 0; FatalError("Could not allocate sigmatch_table"); - return; } } +} +void SigTableSetup(void) +{ DetectSidRegister(); DetectPriorityRegister(); DetectPrefilterRegister(); diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index c9134c77b83a..db4cd957af9d 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -338,6 +338,7 @@ extern int DETECT_TBLSIZE_IDX; #define DETECT_TBLSIZE_STEP 256 int SigTableList(const char *keyword); void SigTableCleanup(void); +void SigTableInit(void); void SigTableSetup(void); void SigTableRegisterTests(void); diff --git a/src/flow-callbacks.c b/src/flow-callbacks.c new file mode 100644 index 000000000000..30e703c3efb3 --- /dev/null +++ b/src/flow-callbacks.c @@ -0,0 +1,129 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "flow-callbacks.h" + +typedef struct FlowInitCallback_ { + SCFlowInitCallbackFn Callback; + void *user; + struct FlowInitCallback_ *next; +} FlowInitCallback; + +static FlowInitCallback *init_callbacks = NULL; + +typedef struct FlowUpdateCallback_ { + SCFlowUpdateCallbackFn Callback; + void *user; + struct FlowUpdateCallback_ *next; +} FlowUpdateCallback; + +static FlowUpdateCallback *update_callbacks = NULL; + +typedef struct FlowFinishCallback_ { + SCFlowFinishCallbackFn Callback; + void *user; + struct FlowFinishCallback_ *next; +} FlowFinishCallback; + +static FlowFinishCallback *finish_callbacks = NULL; + +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user) +{ + FlowInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + FlowInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p) +{ + FlowInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user) +{ + FlowUpdateCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (update_callbacks == NULL) { + update_callbacks = cb; + } else { + FlowUpdateCallback *current = update_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p) +{ + FlowUpdateCallback *cb = update_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, p, cb->user); + cb = cb->next; + } +} + +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user) +{ + FlowFinishCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (finish_callbacks == NULL) { + finish_callbacks = cb; + } else { + FlowFinishCallback *current = finish_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f) +{ + FlowFinishCallback *cb = finish_callbacks; + while (cb != NULL) { + cb->Callback(tv, f, cb->user); + cb = cb->next; + } +} diff --git a/src/flow-callbacks.h b/src/flow-callbacks.h new file mode 100644 index 000000000000..4c694807753f --- /dev/null +++ b/src/flow-callbacks.h @@ -0,0 +1,121 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_FLOW_CALLBACKS_H +#define SURICATA_FLOW_CALLBACKS_H + +#include "suricata-common.h" +#include "flow.h" + +/** \brief Function type for flow initialization callbacks. + * + * Once registered with SCFlowRegisterInitCallback, this function will + * be called every time a flow is initialized, or in other words, + * every time Suricata picks up a flow. + * + * \param tv The ThreadVars data structure for the thread creating the + * flow. + * \param f The newly initialized flow. + * \param p The packet related to creating the new flow. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowInitCallbackFn)(ThreadVars *tv, Flow *f, const Packet *p, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterInitCallback(SCFlowInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunInitCallbacks(ThreadVars *tv, Flow *f, const Packet *p); + +/** \brief Function type for flow update callbacks. + * + * Once registered with SCFlowRegisterUpdateCallback, this function + * will be called every time a flow is updated by a packet (basically + * everytime a packet is seen on a flow). + * + * \param tv The ThreadVars data structure for the thread updating the + * flow. + * \param f The flow being updated. + * \param p The packet responsible for the flow update. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowUpdateCallbackFn)(ThreadVars *tv, Flow *f, Packet *p, void *user); + +/** \brief Register a flow update callback. + * + * Register a user provided function to be called everytime a flow is + * updated. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterUpdateCallback(SCFlowUpdateCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow update callbacks. + */ +void SCFlowRunUpdateCallbacks(ThreadVars *tv, Flow *f, Packet *p); + +/** \brief Function type for flow finish callbacks. + * + * Once registered with SCFlowRegisterFinshCallback, this function + * will be called when Suricata is done with a flow. + * + * \param tv The ThreadVars data structure for the thread finishing + * the flow. + * \param f The flow being finshed. + * \param user The user data provided during callback registration. + */ +typedef void (*SCFlowFinishCallbackFn)(ThreadVars *tv, Flow *f, void *user); + +/** \brief Register a flow init callback. + * + * Register a user provided function to be called every time a flow is + * finished. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCFlowRegisterFinishCallback(SCFlowFinishCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCFlowRunFinishCallbacks(ThreadVars *tv, Flow *f); + +#endif /* SURICATA_FLOW_CALLBACKS_H */ diff --git a/src/flow-hash.c b/src/flow-hash.c index ddab01cd5b69..fcd957c72e27 100644 --- a/src/flow-hash.c +++ b/src/flow-hash.c @@ -38,6 +38,7 @@ #include "flow-storage.h" #include "flow-timeout.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "app-layer-parser.h" #include "util-time.h" @@ -781,7 +782,7 @@ static Flow *TcpReuseReplace(ThreadVars *tv, FlowLookupStruct *fls, FlowBucket * fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -886,7 +887,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* got one, now lock, initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -951,7 +952,7 @@ Flow *FlowGetFlowFromHash(ThreadVars *tv, FlowLookupStruct *fls, Packet *p, Flow fb->head = f; /* initialize and return */ - FlowInit(f, p); + FlowInit(tv, f, p); f->flow_hash = hash; f->fb = fb; FlowUpdateState(f, FLOW_STATE_NEW); @@ -1242,6 +1243,7 @@ static Flow *FlowGetUsedFlow(ThreadVars *tv, DecodeThreadVars *dtv, const SCTime } #endif + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); /* leave locked */ diff --git a/src/flow-manager.c b/src/flow-manager.c index 05b791ee612e..9da986b22df6 100644 --- a/src/flow-manager.c +++ b/src/flow-manager.c @@ -39,6 +39,7 @@ #include "flow-manager.h" #include "flow-storage.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp.h" #include "stream-tcp-cache.h" @@ -1059,7 +1060,7 @@ static void Recycler(ThreadVars *tv, FlowRecyclerThreadData *ftd, Flow *f) StatsDecr(tv, ftd->counter_tcp_active_sessions); } StatsDecr(tv, ftd->counter_flow_active); - + SCFlowRunFinishCallbacks(tv, f); FlowClearMemory(f, f->protomap); FLOWLOCK_UNLOCK(f); } diff --git a/src/flow-util.c b/src/flow-util.c index 7e11da41f527..31e22b9341ac 100644 --- a/src/flow-util.c +++ b/src/flow-util.c @@ -29,6 +29,7 @@ #include "flow.h" #include "flow-private.h" #include "flow-util.h" +#include "flow-callbacks.h" #include "flow-var.h" #include "app-layer.h" @@ -142,7 +143,7 @@ static inline void FlowSetICMPv6CounterPart(Flow *f) /* initialize the flow from the first packet * we see from it. */ -void FlowInit(Flow *f, const Packet *p) +void FlowInit(ThreadVars *tv, Flow *f, const Packet *p) { SCEnter(); SCLogDebug("flow %p", f); @@ -203,6 +204,8 @@ void FlowInit(Flow *f, const Packet *p) FlowSetStorageById(f, MacSetGetFlowStorageID(), ms); } + SCFlowRunInitCallbacks(tv, f, p); + SCReturn; } diff --git a/src/flow-util.h b/src/flow-util.h index 2d813bd9ee4d..368c955d876a 100644 --- a/src/flow-util.h +++ b/src/flow-util.h @@ -140,7 +140,7 @@ Flow *FlowAlloc(void); void FlowFree(Flow *); uint8_t FlowGetProtoMapping(uint8_t); -void FlowInit(Flow *, const Packet *); +void FlowInit(ThreadVars *, Flow *, const Packet *); uint8_t FlowGetReverseProtoMapping(uint8_t rproto); /* flow end counter logic */ diff --git a/src/flow.c b/src/flow.c index 7bfa80ea0a9b..aea79d23bf08 100644 --- a/src/flow.c +++ b/src/flow.c @@ -44,6 +44,7 @@ #include "flow-storage.h" #include "flow-bypass.h" #include "flow-spare-pool.h" +#include "flow-callbacks.h" #include "stream-tcp-private.h" @@ -503,6 +504,8 @@ void FlowHandlePacketUpdate(Flow *f, Packet *p, ThreadVars *tv, DecodeThreadVars SCLogDebug("setting FLOW_NOPAYLOAD_INSPECTION flag on flow %p", f); DecodeSetNoPayloadInspectionFlag(p); } + + SCFlowRunUpdateCallbacks(tv, f, p); } /** \brief Entry point for packet flow handling diff --git a/src/output-eve-stream.c b/src/output-eve-stream.c index fcdf0c2e5c0b..4b44d86835e7 100644 --- a/src/output-eve-stream.c +++ b/src/output-eve-stream.c @@ -425,7 +425,7 @@ static int EveStreamLogger(ThreadVars *tv, void *thread_data, const Packet *p) /* Close stream. */ jb_close(js); - OutputJsonBuilderBuffer(js, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, td->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-eve.c b/src/output-eve.c index d0d775cba7f7..2c67f3b6c38a 100644 --- a/src/output-eve.c +++ b/src/output-eve.c @@ -15,11 +15,50 @@ * 02110-1301, USA. */ +#include "suricata-common.h" #include "output-eve.h" #include "util-debug.h" +#include "rust.h" + +typedef struct EveUserCallback_ { + SCEveUserCallbackFn Callback; + void *user; + struct EveUserCallback_ *next; +} EveUserCallback; + +static EveUserCallback *eve_user_callbacks = NULL; static TAILQ_HEAD(, SCEveFileType_) output_types = TAILQ_HEAD_INITIALIZER(output_types); +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user) +{ + EveUserCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (eve_user_callbacks == NULL) { + eve_user_callbacks = cb; + } else { + EveUserCallback *current = eve_user_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb) +{ + EveUserCallback *cb = eve_user_callbacks; + while (cb != NULL) { + cb->Callback(tv, p, f, jb, cb->user); + cb = cb->next; + } +} + static bool IsBuiltinTypeName(const char *name) { const char *builtin[] = { diff --git a/src/output-eve.h b/src/output-eve.h index 7046c7b98005..7e55ce28f8e2 100644 --- a/src/output-eve.h +++ b/src/output-eve.h @@ -31,6 +31,7 @@ #define SURICATA_OUTPUT_EVE_H #include "suricata-common.h" +#include "rust.h" #include "conf.h" typedef uint32_t ThreadId; @@ -173,4 +174,46 @@ bool SCRegisterEveFileType(SCEveFileType *); SCEveFileType *SCEveFindFileType(const char *name); +/** \brief Function type for EVE callbacks. + * + * The function type for callbacks registered with + * SCEveRegisterCallback. This function will be called with the + * JsonBuilder just prior to the top-level object being closed. New + * fields maybe added, however there is no way to alter existing + * objects already added to the JsonBuilder. + * + * \param tv The ThreadVars for the thread performing the logging. + * \param p Packet if available. + * \param f Flow if available. + * \param user User data provided during callback registration. + */ +typedef void (*SCEveUserCallbackFn)( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb, void *user); + +/** \brief Register a callback for adding extra information to EVE logs. + * + * Allow users to register a callback for each EVE log. The callback + * is called just before the root object on the JsonBuilder is to be + * closed. + * + * New objects and fields can be append, but exist entries cannot be modified. + * + * Packet and Flow will be provided if available, but will other be + * NULL. + * + * Limitations: At this time the callbacks will only be called for EVE + * loggers that use JsonBuilder, notably this means it won't be called + * for stats records at this time. + * + * \returns true if callback is registered, false is not due to memory + * allocation error. + */ +bool SCEveRegisterCallback(SCEveUserCallbackFn fn, void *user); + +/** \internal + * + * Run EVE callbacks. + */ +void SCEveRunCallbacks(ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *jb); + #endif diff --git a/src/output-json-alert.c b/src/output-json-alert.c index 7822cc798045..91a55828a7a1 100644 --- a/src/output-json-alert.c +++ b/src/output-json-alert.c @@ -757,7 +757,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) EveAddVerdict(jb, p); } - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } @@ -767,7 +767,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p) CreateEveHeader(p, LOG_DIR_PACKET, "packet", NULL, json_output_ctx->eve_ctx); if (unlikely(packetjs != NULL)) { EvePacket(p, packetjs, 0); - OutputJsonBuilderBuffer(packetjs, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, packetjs, aft->ctx); jb_free(packetjs); } } @@ -801,7 +801,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const AlertJsonHeader(p, pa, jb, json_output_ctx->flags, NULL, NULL); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); } diff --git a/src/output-json-anomaly.c b/src/output-json-anomaly.c index 241cb974a758..00f82fa3685e 100644 --- a/src/output-json-anomaly.c +++ b/src/output-json-anomaly.c @@ -143,16 +143,16 @@ static int AnomalyDecodeEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, EvePacket(p, js, GET_PKT_LEN(p) < 32 ? GET_PKT_LEN(p) : 32); } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } return TM_ECODE_OK; } -static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, - const Packet *p, AppLayerDecoderEvents *decoder_events, - bool is_pktlayer, const char *layer, uint64_t tx_id) +static int AnomalyAppLayerDecoderEventJson(ThreadVars *tv, JsonAnomalyLogThread *aft, + const Packet *p, AppLayerDecoderEvents *decoder_events, bool is_pktlayer, const char *layer, + uint64_t tx_id) { const char *alprotoname = AppLayerGetProtoName(p->flow->alproto); @@ -201,7 +201,7 @@ static int AnomalyAppLayerDecoderEventJson(JsonAnomalyLogThread *aft, /* anomaly */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); /* Current implementation assumes a single owner for this value */ @@ -223,8 +223,7 @@ static int JsonAnomalyTxLogger(ThreadVars *tv, void *thread_data, const Packet * decoder_events = AppLayerParserGetEventsByTx(f->proto, f->alproto, tx); if (decoder_events && decoder_events->event_last_logged < decoder_events->cnt) { SCLogDebug("state %p, tx: %p, tx_id: %"PRIu64, state, tx, tx_id); - AnomalyAppLayerDecoderEventJson(aft, p, decoder_events, false, - "proto_parser", tx_id); + AnomalyAppLayerDecoderEventJson(tv, aft, p, decoder_events, false, "proto_parser", tx_id); } return TM_ECODE_OK; } @@ -255,8 +254,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * if (aft->json_output_ctx->flags & LOG_JSON_APPLAYER_TYPE) { /* app layer proto detect events */ if (rc == TM_ECODE_OK && AnomalyHasPacketAppLayerEvents(p)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, p->app_layer_events, - true, "proto_detect", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, p->app_layer_events, true, "proto_detect", TX_ID_UNUSED); } /* parser state events */ @@ -264,8 +263,8 @@ static int AnomalyJson(ThreadVars *tv, JsonAnomalyLogThread *aft, const Packet * SCLogDebug("Checking for anomaly events; alproto %d", p->flow->alproto); AppLayerDecoderEvents *parser_events = AppLayerParserGetDecoderEvents(p->flow->alparser); if (parser_events && (parser_events->event_last_logged < parser_events->cnt)) { - rc = AnomalyAppLayerDecoderEventJson(aft, p, parser_events, - false, "parser", TX_ID_UNUSED); + rc = AnomalyAppLayerDecoderEventJson( + tv, aft, p, parser_events, false, "parser", TX_ID_UNUSED); } } } diff --git a/src/output-json-arp.c b/src/output-json-arp.c index 0490c6b54d1e..87a80d8cdb0b 100644 --- a/src/output-json-arp.c +++ b/src/output-json-arp.c @@ -90,7 +90,7 @@ static int JsonArpLogger(ThreadVars *tv, void *thread_data, const Packet *p) JSONFormatAndAddMACAddr(jb, "dest_mac", arph->dest_mac, false); jb_set_string(jb, "dest_ip", dstip); jb_close(jb); /* arp */ - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dcerpc.c b/src/output-json-dcerpc.c index 17e0199ed727..3b3bff90feac 100644 --- a/src/output-json-dcerpc.c +++ b/src/output-json-dcerpc.c @@ -47,7 +47,7 @@ static int JsonDCERPCLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-dhcp.c b/src/output-json-dhcp.c index 9c7d9dff9230..a4a4a29990e8 100644 --- a/src/output-json-dhcp.c +++ b/src/output-json-dhcp.c @@ -72,7 +72,7 @@ static int JsonDHCPLogger(ThreadVars *tv, void *thread_data, rs_dhcp_logger_log(ctx->rs_logger, tx, js); - OutputJsonBuilderBuffer(js, thread->thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->thread); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-dnp3.c b/src/output-json-dnp3.c index 53cecd78a1aa..ea557ff206a8 100644 --- a/src/output-json-dnp3.c +++ b/src/output-json-dnp3.c @@ -246,7 +246,7 @@ static int JsonDNP3LoggerToServer(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogRequest(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); @@ -267,7 +267,7 @@ static int JsonDNP3LoggerToClient(ThreadVars *tv, void *thread_data, jb_open_object(js, "dnp3"); JsonDNP3LogResponse(js, tx); jb_close(js); - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-dns.c b/src/output-json-dns.c index 3954da2336dc..cb60a4509a32 100644 --- a/src/output-json-dns.c +++ b/src/output-json-dns.c @@ -331,7 +331,7 @@ static int JsonDoh2Logger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } out: if (r || r2) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); return TM_ECODE_OK; @@ -363,7 +363,7 @@ static int JsonDnsLoggerToServer(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -392,7 +392,7 @@ static int JsonDnsLoggerToClient(ThreadVars *tv, void *thread_data, jb_set_int(jb, "version", 2); SCDnsLogJsonAnswer(txptr, td->dnslog_ctx->flags, jb); jb_close(jb); - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); jb_free(jb); } @@ -432,7 +432,7 @@ static int JsonDnsLogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo } if (SCDnsLogJson(txptr, td->dnslog_ctx->flags, jb)) { - OutputJsonBuilderBuffer(jb, td->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, td->ctx); } jb_free(jb); } diff --git a/src/output-json-drop.c b/src/output-json-drop.c index b82c632daf65..1ac27a209d2a 100644 --- a/src/output-json-drop.c +++ b/src/output-json-drop.c @@ -85,7 +85,7 @@ static int g_droplog_flows_start = 1; * * \return return TM_ECODE_OK on success */ -static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) +static int DropLogJSON(ThreadVars *tv, JsonDropLogThread *aft, const Packet *p) { JsonDropOutputCtx *drop_ctx = aft->drop_ctx; @@ -191,7 +191,7 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p) } } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return TM_ECODE_OK; @@ -326,7 +326,7 @@ static OutputInitResult JsonDropLogInitCtxSub(ConfNode *conf, OutputCtx *parent_ static int JsonDropLogger(ThreadVars *tv, void *thread_data, const Packet *p) { JsonDropLogThread *td = thread_data; - int r = DropLogJSON(td, p); + int r = DropLogJSON(tv, td, p); if (r < 0) return -1; diff --git a/src/output-json-file.c b/src/output-json-file.c index 509ae488bbee..e1f33893806e 100644 --- a/src/output-json-file.c +++ b/src/output-json-file.c @@ -213,8 +213,8 @@ JsonBuilder *JsonBuildFileInfoRecord(const Packet *p, const File *ff, void *tx, * \internal * \brief Write meta data on a single line json record */ -static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const File *ff, void *tx, - const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) +static void FileWriteJsonRecord(ThreadVars *tv, JsonFileLogThread *aft, const Packet *p, + const File *ff, void *tx, const uint64_t tx_id, uint8_t dir, OutputJsonCtx *eve_ctx) { HttpXFFCfg *xff_cfg = aft->filelog_ctx->xff_cfg != NULL ? aft->filelog_ctx->xff_cfg : aft->filelog_ctx->parent_xff_cfg; @@ -223,7 +223,7 @@ static void FileWriteJsonRecord(JsonFileLogThread *aft, const Packet *p, const F return; } - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); } @@ -237,7 +237,7 @@ static int JsonFileLogger(ThreadVars *tv, void *thread_data, const Packet *p, co SCLogDebug("ff %p", ff); - FileWriteJsonRecord(aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); + FileWriteJsonRecord(tv, aft, p, ff, tx, tx_id, dir, aft->filelog_ctx->eve_ctx); return 0; } diff --git a/src/output-json-flow.c b/src/output-json-flow.c index f7826734f0cb..051d530fb1ef 100644 --- a/src/output-json-flow.c +++ b/src/output-json-flow.c @@ -340,7 +340,7 @@ static int JsonFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) EveFlowLogJSON(thread, jb, f); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, NULL, f, jb, thread); jb_free(jb); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-frame.c b/src/output-json-frame.c index 09ec4aaab110..90224240f43f 100644 --- a/src/output-json-frame.c +++ b/src/output-json-frame.c @@ -287,8 +287,8 @@ void FrameJsonLogOneFrame(const uint8_t ipproto, const Frame *frame, Flow *f, jb_close(jb); } -static int FrameJsonUdp( - JsonFrameLogThread *aft, const Packet *p, Flow *f, FramesContainer *frames_container) +static int FrameJsonUdp(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p, Flow *f, + FramesContainer *frames_container) { FrameJsonOutputCtx *json_output_ctx = aft->json_output_ctx; @@ -315,7 +315,7 @@ static int FrameJsonUdp( jb_set_string(jb, "app_proto", AppProtoToString(f->alproto)); FrameJsonLogOneFrame(IPPROTO_UDP, frame, p->flow, NULL, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } @@ -333,7 +333,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) return TM_ECODE_OK; if (p->proto == IPPROTO_UDP) { - return FrameJsonUdp(aft, p, p->flow, frames_container); + return FrameJsonUdp(tv, aft, p, p->flow, frames_container); } BUG_ON(p->proto != IPPROTO_TCP); @@ -387,7 +387,7 @@ static int FrameJson(ThreadVars *tv, JsonFrameLogThread *aft, const Packet *p) jb_set_string(jb, "app_proto", AppProtoToString(p->flow->alproto)); FrameJsonLogOneFrame(IPPROTO_TCP, frame, p->flow, stream, p, jb, aft->payload_buffer); - OutputJsonBuilderBuffer(jb, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, aft->ctx); jb_free(jb); frame->flags |= FRAME_FLAG_LOGGED; } else if (frame != NULL) { diff --git a/src/output-json-http.c b/src/output-json-http.c index 0c5b875ee9ad..b45be9a45b6b 100644 --- a/src/output-json-http.c +++ b/src/output-json-http.c @@ -493,7 +493,7 @@ static int JsonHttpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl } } - OutputJsonBuilderBuffer(js, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, jhl->ctx); jb_free(js); SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-ike.c b/src/output-json-ike.c index 470026fde13b..a13ef0e1d944 100644 --- a/src/output-json-ike.c +++ b/src/output-json-ike.c @@ -90,7 +90,7 @@ static int JsonIKELogger(ThreadVars *tv, void *thread_data, const Packet *p, Flo goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-metadata.c b/src/output-json-metadata.c index f97547c551b4..2602e4b9b3ef 100644 --- a/src/output-json-metadata.c +++ b/src/output-json-metadata.c @@ -74,7 +74,7 @@ static int MetadataJson(ThreadVars *tv, OutputJsonThreadCtx *aft, const Packet * if (!aft->ctx->cfg.include_metadata) { EveAddMetadata(p, p->flow, js); } - OutputJsonBuilderBuffer(js, aft); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-mqtt.c b/src/output-json-mqtt.c index 66cf67a0334b..c912ddcc1835 100644 --- a/src/output-json-mqtt.c +++ b/src/output-json-mqtt.c @@ -85,7 +85,7 @@ static int JsonMQTTLogger(ThreadVars *tv, void *thread_data, if (!rs_mqtt_logger_log(tx, thread->mqttlog_ctx->flags, thread->mqttlog_ctx->max_log_len, js)) goto error; - OutputJsonBuilderBuffer(js, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread->ctx); jb_free(js); return TM_ECODE_OK; diff --git a/src/output-json-netflow.c b/src/output-json-netflow.c index 2e359bb909c5..e448ecd33bc4 100644 --- a/src/output-json-netflow.c +++ b/src/output-json-netflow.c @@ -276,7 +276,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToServer(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOSERVER); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); /* only log a response record if we actually have seen response packets */ @@ -286,7 +286,7 @@ static int JsonNetFlowLogger(ThreadVars *tv, void *thread_data, Flow *f) return TM_ECODE_OK; NetFlowLogEveToClient(jb, f); EveAddCommonOptions(&jhl->ctx->cfg, NULL, f, jb, LOG_DIR_FLOW_TOCLIENT); - OutputJsonBuilderBuffer(jb, jhl); + OutputJsonBuilderBuffer(tv, NULL, f, jb, jhl); jb_free(jb); } SCReturnInt(TM_ECODE_OK); diff --git a/src/output-json-nfs.c b/src/output-json-nfs.c index 72274a6b7865..0b08c0e5105d 100644 --- a/src/output-json-nfs.c +++ b/src/output-json-nfs.c @@ -94,7 +94,7 @@ static int JsonNFSLogger(ThreadVars *tv, void *thread_data, jb_close(jb); MemBufferReset(thread->buffer); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; } diff --git a/src/output-json-pgsql.c b/src/output-json-pgsql.c index 71bcd10f071d..9cba28d25d4e 100644 --- a/src/output-json-pgsql.c +++ b/src/output-json-pgsql.c @@ -80,7 +80,7 @@ static int JsonPgsqlLogger(ThreadVars *tv, void *thread_data, const Packet *p, F goto error; } - OutputJsonBuilderBuffer(jb, thread->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread->ctx); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smb.c b/src/output-json-smb.c index 279ee772e8f0..4be1fce93e72 100644 --- a/src/output-json-smb.c +++ b/src/output-json-smb.c @@ -59,7 +59,7 @@ static int JsonSMBLogger(ThreadVars *tv, void *thread_data, } jb_close(jb); - OutputJsonBuilderBuffer(jb, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, thread); jb_free(jb); return TM_ECODE_OK; diff --git a/src/output-json-smtp.c b/src/output-json-smtp.c index bddbc4a9fcc4..592645cb3c09 100644 --- a/src/output-json-smtp.c +++ b/src/output-json-smtp.c @@ -85,7 +85,7 @@ static int JsonSmtpLogger(ThreadVars *tv, void *thread_data, const Packet *p, Fl jb_close(jb); EveEmailLogJson(jhl, jb, p, f, state, tx, tx_id); - OutputJsonBuilderBuffer(jb, jhl->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, jb, jhl->ctx); jb_free(jb); diff --git a/src/output-json-tls.c b/src/output-json-tls.c index aa24b3380a0d..c4ba0e249e62 100644 --- a/src/output-json-tls.c +++ b/src/output-json-tls.c @@ -501,7 +501,7 @@ static int JsonTlsLogger(ThreadVars *tv, void *thread_data, const Packet *p, /* Close the tls object. */ jb_close(js); - OutputJsonBuilderBuffer(js, aft->ctx); + OutputJsonBuilderBuffer(tv, p, p->flow, js, aft->ctx); jb_free(js); return 0; diff --git a/src/output-json.c b/src/output-json.c index 1f411cc110b8..18376fd428a5 100644 --- a/src/output-json.c +++ b/src/output-json.c @@ -955,7 +955,8 @@ int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer) return 0; } -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx) { LogFileCtx *file_ctx = ctx->file_ctx; MemBuffer **buffer = &ctx->buffer; @@ -967,6 +968,8 @@ int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx) jb_set_string(js, "pcap_filename", PcapFileGetFilename()); } + SCEveRunCallbacks(tv, p, f, js); + jb_close(js); MemBufferReset(*buffer); diff --git a/src/output-json.h b/src/output-json.h index 761064f7e10a..89597e616a0f 100644 --- a/src/output-json.h +++ b/src/output-json.h @@ -103,7 +103,8 @@ JsonBuilder *CreateEveHeader(const Packet *p, enum OutputJsonLogDirection dir, JsonBuilder *CreateEveHeaderWithTxId(const Packet *p, enum OutputJsonLogDirection dir, const char *event_type, JsonAddrInfo *addr, uint64_t tx_id, OutputJsonCtx *eve_ctx); int OutputJSONBuffer(json_t *js, LogFileCtx *file_ctx, MemBuffer **buffer); -int OutputJsonBuilderBuffer(JsonBuilder *js, OutputJsonThreadCtx *ctx); +int OutputJsonBuilderBuffer( + ThreadVars *tv, const Packet *p, Flow *f, JsonBuilder *js, OutputJsonThreadCtx *ctx); OutputInitResult OutputJsonInitCtx(ConfNode *); OutputInitResult OutputJsonLogInitSub(ConfNode *conf, OutputCtx *parent_ctx); diff --git a/src/output.c b/src/output.c index 002f33b5abc6..b99897509c0f 100644 --- a/src/output.c +++ b/src/output.c @@ -927,7 +927,7 @@ static int JsonGenericLogger(ThreadVars *tv, void *thread_data, const Packet *p, goto error; } - OutputJsonBuilderBuffer(js, thread); + OutputJsonBuilderBuffer(tv, p, p->flow, js, thread); jb_free(js); return TM_ECODE_OK; diff --git a/src/runmode-unittests.c b/src/runmode-unittests.c index e116e86d5be5..35780ab101a0 100644 --- a/src/runmode-unittests.c +++ b/src/runmode-unittests.c @@ -240,6 +240,7 @@ void RunUnittests(int list_unittests, const char *regex_arg) AppLayerSetup(); /* hardcoded initialization code */ + SigTableInit(); SigTableSetup(); /* load the rule keywords */ TmqhSetup(); diff --git a/src/suricata.c b/src/suricata.c index 6bdd6edb90f6..ee9dfc0b5b69 100644 --- a/src/suricata.c +++ b/src/suricata.c @@ -2670,6 +2670,8 @@ int PostConfLoadedSetup(SCInstance *suri) MacSetRegisterFlowStorage(); + SigTableInit(); + #ifdef HAVE_PLUGINS SCPluginsLoad(suri->capture_plugin_name, suri->capture_plugin_args); #endif diff --git a/src/tests/fuzz/fuzz_siginit.c b/src/tests/fuzz/fuzz_siginit.c index a98148cfac8d..a50e1fd67ebf 100644 --- a/src/tests/fuzz/fuzz_siginit.c +++ b/src/tests/fuzz/fuzz_siginit.c @@ -27,6 +27,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) MpmTableSetup(); SpmTableSetup(); EngineModeSetIDS(); + SigTableInit(); SigTableSetup(); } if (cnt++ == 1024) { diff --git a/src/thread-callbacks.c b/src/thread-callbacks.c new file mode 100644 index 000000000000..ede35d7107ce --- /dev/null +++ b/src/thread-callbacks.c @@ -0,0 +1,55 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "thread-callbacks.h" + +typedef struct ThreadInitCallback_ { + SCThreadInitCallbackFn Callback; + void *user; + struct ThreadInitCallback_ *next; +} ThreadInitCallback; + +static ThreadInitCallback *init_callbacks = NULL; + +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user) +{ + ThreadInitCallback *cb = SCCalloc(1, sizeof(*cb)); + if (cb == NULL) { + return false; + } + cb->Callback = fn; + cb->user = user; + if (init_callbacks == NULL) { + init_callbacks = cb; + } else { + ThreadInitCallback *current = init_callbacks; + while (current->next != NULL) { + current = current->next; + } + current->next = cb; + } + return true; +} + +void SCThreadRunInitCallbacks(ThreadVars *tv) +{ + ThreadInitCallback *cb = init_callbacks; + while (cb != NULL) { + cb->Callback(tv, cb->user); + cb = cb->next; + } +} diff --git a/src/thread-callbacks.h b/src/thread-callbacks.h new file mode 100644 index 000000000000..5bcd638bf86b --- /dev/null +++ b/src/thread-callbacks.h @@ -0,0 +1,54 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#ifndef SURICATA_THREAD_CALLBACKS_H +#define SURICATA_THREAD_CALLBACKS_H + +#include "suricata-common.h" +#include "threadvars.h" + +/** \brief Function type for thread intialization callbacks. + * + * Once registered by SCThreadRegisterInitCallback, this function will + * be called for every thread being initialized during Suricata + * startup. + * + * \param tv The ThreadVars struct that has just been initialized. + * \param user The user data provided when registering the callback. + */ +typedef void (*SCThreadInitCallbackFn)(ThreadVars *tv, void *user); + +/** \brief Register a thread init callback. + * + * Register a user provided function to be called every time a thread is + * initialized for use. + * + * \param fn Pointer to function to be called + * \param user Additional user data to be passed to callback + * + * \returns true if callback was registered, otherwise false if the + * callback could not be registered due to memory allocation error. + */ +bool SCThreadRegisterInitCallback(SCThreadInitCallbackFn fn, void *user); + +/** \internal + * + * Run all registered flow init callbacks. + */ +void SCThreadRunInitCallbacks(ThreadVars *tv); + +#endif /* SURICATA_THREAD_CALLBACKS_H */ diff --git a/src/thread-storage.c b/src/thread-storage.c new file mode 100644 index 000000000000..977f4fde9752 --- /dev/null +++ b/src/thread-storage.c @@ -0,0 +1,212 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include "suricata-common.h" +#include "thread-storage.h" +#include "util-storage.h" +#include "util-unittest.h" + +const StorageEnum storage_type = STORAGE_THREAD; + +unsigned int ThreadStorageSize(void) +{ + return StorageGetSize(storage_type); +} + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id) +{ + return StorageGetById(tv->storage, storage_type, id.id); +} + +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr) +{ + return StorageSetById(tv->storage, storage_type, id.id, ptr); +} + +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id) +{ + return StorageAllocByIdPrealloc(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id) +{ + StorageFreeById(tv->storage, storage_type, id.id); +} + +void ThreadFreeStorage(ThreadVars *tv) +{ + if (ThreadStorageSize() > 0) + StorageFreeAll(tv->storage, storage_type); +} + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)) +{ + int id = StorageRegister(storage_type, name, size, Alloc, Free); + ThreadStorageId tsi = { .id = id }; + return tsi; +} + +#ifdef UNITTESTS + +static void *StorageTestAlloc(unsigned int size) +{ + return SCCalloc(1, size); +} + +static void StorageTestFree(void *x) +{ + SCFree(x); +} + +static int ThreadStorageTest01(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", 8, StorageTestAlloc, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("variable", 24, StorageTestAlloc, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = + ThreadStorageRegister("store", sizeof(void *), StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id2); + FAIL_IF_NOT_NULL(ptr); + + ptr = ThreadGetStorageById(tv, id3); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = ThreadAllocStorageById(tv, id1); + FAIL_IF_NULL(ptr1a); + + void *ptr2a = ThreadAllocStorageById(tv, id2); + FAIL_IF_NULL(ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + SCFree(tv); + PASS; +} + +static int ThreadStorageTest02(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} + +static int ThreadStorageTest03(void) +{ + StorageInit(); + + ThreadStorageId id1 = ThreadStorageRegister("test1", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id1.id < 0); + + ThreadStorageId id2 = ThreadStorageRegister("test2", sizeof(void *), NULL, StorageTestFree); + FAIL_IF(id2.id < 0); + + ThreadStorageId id3 = ThreadStorageRegister("test3", 32, StorageTestAlloc, StorageTestFree); + FAIL_IF(id3.id < 0); + + FAIL_IF(StorageFinalize() < 0); + + ThreadVars *tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); + FAIL_IF_NULL(tv); + + void *ptr = ThreadGetStorageById(tv, id1); + FAIL_IF_NOT_NULL(ptr); + + void *ptr1a = SCMalloc(128); + FAIL_IF_NULL(ptr1a); + + ThreadSetStorageById(tv, id1, ptr1a); + + void *ptr2a = SCMalloc(256); + FAIL_IF_NULL(ptr2a); + + ThreadSetStorageById(tv, id2, ptr2a); + + void *ptr3a = ThreadAllocStorageById(tv, id3); + FAIL_IF_NULL(ptr3a); + + void *ptr1b = ThreadGetStorageById(tv, id1); + FAIL_IF(ptr1a != ptr1b); + + void *ptr2b = ThreadGetStorageById(tv, id2); + FAIL_IF(ptr2a != ptr2b); + + void *ptr3b = ThreadGetStorageById(tv, id3); + FAIL_IF(ptr3a != ptr3b); + + ThreadFreeStorage(tv); + StorageCleanup(); + PASS; +} +#endif + +void RegisterThreadStorageTests(void) +{ +#ifdef UNITTESTS + UtRegisterTest("ThreadStorageTest01", ThreadStorageTest01); + UtRegisterTest("ThreadStorageTest02", ThreadStorageTest02); + UtRegisterTest("ThreadStorageTest03", ThreadStorageTest03); +#endif +} diff --git a/src/thread-storage.h b/src/thread-storage.h new file mode 100644 index 000000000000..5dd22570b0fa --- /dev/null +++ b/src/thread-storage.h @@ -0,0 +1,45 @@ +/* Copyright (C) 2024 Open Information Security Foundation + * + * You can copy, redistribute or modify this Program under the terms of + * the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * version 2 along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +/** + * Thread wrapper around storage API. + */ + +#ifndef SURICATA_THREAD_STORAGE_H +#define SURICATA_THREAD_STORAGE_H + +#include "threadvars.h" + +typedef struct ThreadStorageId { + int id; +} ThreadStorageId; + +unsigned int ThreadStorageSize(void); + +void *ThreadGetStorageById(const ThreadVars *tv, ThreadStorageId id); +int ThreadSetStorageById(ThreadVars *tv, ThreadStorageId id, void *ptr); +void *ThreadAllocStorageById(ThreadVars *tv, ThreadStorageId id); + +void ThreadFreeStorageById(ThreadVars *tv, ThreadStorageId id); +void ThreadFreeStorage(ThreadVars *tv); + +void RegisterThreadStorageTests(void); + +ThreadStorageId ThreadStorageRegister(const char *name, const unsigned int size, + void *(*Alloc)(unsigned int), void (*Free)(void *)); + +#endif /* SURICATA_THREAD_STORAGE_H */ diff --git a/src/threads.c b/src/threads.c index 1708a8f5cd37..919e6422e32f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -25,6 +25,7 @@ */ #include "suricata-common.h" +#include "thread-storage.h" #include "util-unittest.h" #include "util-debug.h" #include "threads.h" @@ -149,5 +150,6 @@ void ThreadMacrosRegisterTests(void) UtRegisterTest("ThreadMacrosTest03RWLocks", ThreadMacrosTest03RWLocks); UtRegisterTest("ThreadMacrosTest04RWLocks", ThreadMacrosTest04RWLocks); // UtRegisterTest("ThreadMacrosTest05RWLocks", ThreadMacrosTest05RWLocks); + RegisterThreadStorageTests(); #endif /* UNIT TESTS */ } diff --git a/src/threadvars.h b/src/threadvars.h index cebcdb4e3ac1..6f339e9839d5 100644 --- a/src/threadvars.h +++ b/src/threadvars.h @@ -28,6 +28,7 @@ #include "counters.h" #include "packet-queue.h" #include "util-atomic.h" +#include "util-storage.h" struct TmSlot_; @@ -135,6 +136,7 @@ typedef struct ThreadVars_ { struct FlowQueue_ *flow_queue; bool break_loop; + Storage storage[]; } ThreadVars; /** Thread setup flags: */ diff --git a/src/tm-threads.c b/src/tm-threads.c index b0d0f8686ba0..07f9a9390df0 100644 --- a/src/tm-threads.c +++ b/src/tm-threads.c @@ -29,7 +29,9 @@ #include "suricata.h" #include "stream.h" #include "runmodes.h" +#include "thread-callbacks.h" #include "threadvars.h" +#include "thread-storage.h" #include "tm-queues.h" #include "tm-queuehandlers.h" #include "tm-threads.h" @@ -919,7 +921,7 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i SCLogDebug("creating thread \"%s\"...", name); /* XXX create separate function for this: allocate a thread container */ - tv = SCCalloc(1, sizeof(ThreadVars)); + tv = SCCalloc(1, sizeof(ThreadVars) + ThreadStorageSize()); if (unlikely(tv == NULL)) goto error; @@ -1011,6 +1013,8 @@ ThreadVars *TmThreadCreate(const char *name, const char *inq_name, const char *i if (mucond != 0) TmThreadInitMC(tv); + SCThreadRunInitCallbacks(tv); + return tv; error: @@ -1577,6 +1581,8 @@ static void TmThreadFree(ThreadVars *tv) SCLogDebug("Freeing thread '%s'.", tv->name); + ThreadFreeStorage(tv); + if (tv->flow_queue) { BUG_ON(tv->flow_queue->qlen != 0); SCFree(tv->flow_queue); diff --git a/src/util-storage.c b/src/util-storage.c index 02f69a568cd2..bae251432315 100644 --- a/src/util-storage.c +++ b/src/util-storage.c @@ -59,6 +59,8 @@ static const char *StoragePrintType(StorageEnum type) return "ippair"; case STORAGE_DEVICE: return "livedevice"; + case STORAGE_THREAD: + return "thread"; case STORAGE_MAX: return "max"; } diff --git a/src/util-storage.h b/src/util-storage.h index 11d64bdbecbd..fce1f964eb14 100644 --- a/src/util-storage.h +++ b/src/util-storage.h @@ -31,6 +31,7 @@ typedef enum StorageEnum_ { STORAGE_FLOW, STORAGE_IPPAIR, STORAGE_DEVICE, + STORAGE_THREAD, STORAGE_MAX, } StorageEnum;