From 4ec90bd2276afab1c3c3eea82a1fd1216ee0961c Mon Sep 17 00:00:00 2001 From: Philippe Antoine Date: Thu, 30 Nov 2023 14:47:14 +0100 Subject: [PATCH] detect: absent keyword to test absence of sticky buffer Ticket: 2224 It takes an argument to match only if the buffer is absent, or it can still match if the buffer is present, but we test the absence of some content. For multi buffers, absent matches if there are 0 buffers. For file keywords, absent matches if there is no file. --- doc/userguide/rules/payload-keywords.rst | 21 ++++ src/detect-engine-analyzer.c | 9 ++ src/detect-engine-content-inspection.c | 28 ++++- src/detect-engine-content-inspection.h | 6 + src/detect-engine-mpm.c | 3 + src/detect-engine-register.h | 1 + src/detect-engine.c | 12 ++ src/detect-file-data.c | 8 ++ src/detect-filemagic.c | 10 +- src/detect-filename.c | 10 +- src/detect-http-client-body.c | 3 + src/detect-http-header.c | 3 + src/detect-isdataat.c | 136 +++++++++++++++++++++++ src/detect-isdataat.h | 7 ++ src/detect-parse.c | 4 + src/detect.h | 2 + 16 files changed, 259 insertions(+), 4 deletions(-) diff --git a/doc/userguide/rules/payload-keywords.rst b/doc/userguide/rules/payload-keywords.rst index 2e92b7046269..780ad111f7dc 100644 --- a/doc/userguide/rules/payload-keywords.rst +++ b/doc/userguide/rules/payload-keywords.rst @@ -274,6 +274,27 @@ You can also use the negation (!) before isdataat. .. image:: payload-keywords/isdataat1.png +absent +------ + +The keyword ``absent`` checks that a sticky buffer does not exist. +It can be used without any argument to match only on absent buffer : + +Example of ``absent`` in a rule: + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent;` sid:1; rev:1;) + + +It can take an argument "or_else" to match on absent buffer or on what comes next such as negated content, for instance : + +.. container:: example-rule + + alert http any any -> any any (msg:"HTTP request without referer"; :example-rule-emphasis:`http.referer; absent: or_else;` content: !"abc"; sid:1; rev:1;) + +For files (ie ``file.data``), absent means there are no files in the transaction. + bsize ----- diff --git a/src/detect-engine-analyzer.c b/src/detect-engine-analyzer.c index d852792fd3da..dae6f9f2f270 100644 --- a/src/detect-engine-analyzer.c +++ b/src/detect-engine-analyzer.c @@ -39,6 +39,7 @@ #include "detect-pcre.h" #include "detect-bytejump.h" #include "detect-bytetest.h" +#include "detect-isdataat.h" #include "detect-flow.h" #include "detect-tcp-flags.h" #include "detect-tcp-ack.h" @@ -855,6 +856,14 @@ static void DumpMatches(RuleAnalyzer *ctx, JsonBuilder *js, const SigMatchData * jb_close(js); break; } + case DETECT_ABSENT: { + const DetectAbsentData *dad = (const DetectAbsentData *)smd->ctx; + jb_open_object(js, "absent"); + jb_set_bool(js, "or_else", dad->or_else); + jb_close(js); + break; + } + case DETECT_IPOPTS: { const DetectIpOptsData *cd = (const DetectIpOptsData *)smd->ctx; diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c index d4dab42816d5..e43e693b2151 100644 --- a/src/detect-engine-content-inspection.c +++ b/src/detect-engine-content-inspection.c @@ -382,6 +382,13 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, prev_offset); } while(1); + } else if (smd->type == DETECT_ABSENT) { + const DetectAbsentData *id = (DetectAbsentData *)smd->ctx; + if (!id->or_else) { + // we match only on absent buffer + goto no_match; + } + goto match; } else if (smd->type == DETECT_ISDATAAT) { SCLogDebug("inspecting isdataat"); @@ -646,8 +653,7 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx, goto match; } goto no_match_discontinue; - } - else if (smd->type == DETECT_LUA) { + } else if (smd->type == DETECT_LUA) { SCLogDebug("lua starting"); if (DetectLuaMatchBuffer(det_ctx, s, smd, buffer, buffer_len, @@ -758,6 +764,24 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh return false; } +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd) +{ + // we will match on NULL buffers there is one absent + bool absent_data = false; + while (1) { + if (smd->type == DETECT_ABSENT) { + absent_data = true; + break; + } + if (smd->is_last) { + break; + } + // smd does not get reused after this loop + smd++; + } + return absent_data; +} + #ifdef UNITTESTS #include "tests/detect-engine-content-inspection.c" #endif diff --git a/src/detect-engine-content-inspection.h b/src/detect-engine-content-inspection.h index 2c253b77ad3d..fe53086badd9 100644 --- a/src/detect-engine-content-inspection.h +++ b/src/detect-engine-content-inspection.h @@ -68,6 +68,12 @@ bool DetectEngineContentInspectionBuffer(DetectEngineCtx *de_ctx, DetectEngineTh const Signature *s, const SigMatchData *smd, Packet *p, Flow *f, const InspectionBuffer *b, const enum DetectContentInspectionType inspection_mode); +/** \brief tells if we should match on absent buffer, because + * there is an absent keyword being used + * \param smd array of content inspection matches + * \retval bool true to match on absent buffer, false otherwise */ +bool DetectContentInspectionMatchOnAbsentBuffer(const SigMatchData *smd); + void DetectEngineContentInspectionRegisterTests(void); #endif /* SURICATA_DETECT_ENGINE_CONTENT_INSPECTION_H */ diff --git a/src/detect-engine-mpm.c b/src/detect-engine-mpm.c index 5e8687e34686..1c9984ea9541 100644 --- a/src/detect-engine-mpm.c +++ b/src/detect-engine-mpm.c @@ -1125,6 +1125,9 @@ void RetrieveFPForSig(const DetectEngineCtx *de_ctx, Signature *s) } for (SigMatch *sm = s->init_data->buffers[x].head; sm != NULL; sm = sm->next) { + // a buffer with absent keyword cannot be used as fast_pattern + if (sm->type == DETECT_ABSENT) + break; if (sm->type != DETECT_CONTENT) continue; diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h index db4cd957af9d..e5f550be7547 100644 --- a/src/detect-engine-register.h +++ b/src/detect-engine-register.h @@ -93,6 +93,7 @@ enum DetectKeywordId { DETECT_LUA, DETECT_ISDATAAT, DETECT_AL_URILEN, + DETECT_ABSENT, /* end of content inspection */ DETECT_METADATA, diff --git a/src/detect-engine.c b/src/detect-engine.c index 77c25a1cf3a9..f4195739e2c1 100644 --- a/src/detect-engine.c +++ b/src/detect-engine.c @@ -666,6 +666,7 @@ static void AppendAppInspectEngine(DetectEngineCtx *de_ctx, new_engine->sm_list = t->sm_list; new_engine->sm_list_base = t->sm_list_base; new_engine->smd = smd; + new_engine->match_on_null = DetectContentInspectionMatchOnAbsentBuffer(smd); new_engine->progress = t->progress; new_engine->v2 = t->v2; SCLogDebug("sm_list %d new_engine->v2 %p/%p/%p", new_engine->sm_list, new_engine->v2.Callback, @@ -2172,6 +2173,9 @@ uint8_t DetectEngineInspectBufferGeneric(DetectEngineCtx *de_ctx, DetectEngineTh const InspectionBuffer *buffer = engine->v2.GetData(det_ctx, transforms, f, flags, txv, list_id); if (unlikely(buffer == NULL)) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } @@ -2233,6 +2237,14 @@ uint8_t DetectEngineInspectMultiBufferGeneric(DetectEngineCtx *de_ctx, } local_id++; } while (1); + if (local_id == 0) { + // That means we did not get even one buffer value from the multi-buffer + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + } return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-file-data.c b/src/detect-file-data.c index a721c08c7cf9..d976b51c00b4 100644 --- a/src/detect-file-data.c +++ b/src/detect-file-data.c @@ -403,6 +403,14 @@ uint8_t DetectEngineInspectFiledata(DetectEngineCtx *de_ctx, DetectEngineThreadC if (ffc == NULL) { return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } + if (ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } int local_file_id = 0; File *file = ffc->head; diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c index f23434d8666e..0f8d94a7b5b1 100644 --- a/src/detect-filemagic.c +++ b/src/detect-filemagic.c @@ -307,7 +307,15 @@ static uint8_t DetectEngineInspectFilemagic(DetectEngineCtx *de_ctx, DetectEngin AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-filename.c b/src/detect-filename.c index f75fdbd680fe..ef144cf44086 100644 --- a/src/detect-filename.c +++ b/src/detect-filename.c @@ -244,7 +244,15 @@ static uint8_t DetectEngineInspectFilename(DetectEngineCtx *de_ctx, DetectEngine AppLayerGetFileState files = AppLayerParserGetTxFiles(f, txv, flags); FileContainer *ffc = files.fc; - if (ffc == NULL) { + if (ffc == NULL || ffc->head == NULL) { + const bool eof = (AppLayerParserGetStateProgress(f->proto, f->alproto, txv, flags) > + engine->progress); + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } + if (ffc != NULL) { + return DETECT_ENGINE_INSPECT_SIG_NO_MATCH; + } return DETECT_ENGINE_INSPECT_SIG_CANT_MATCH_FILES; } diff --git a/src/detect-http-client-body.c b/src/detect-http-client-body.c index 5e5604ea594d..7747b61b858b 100644 --- a/src/detect-http-client-body.c +++ b/src/detect-http-client-body.c @@ -312,6 +312,9 @@ static uint8_t DetectEngineInspectBufferHttpBody(DetectEngineCtx *de_ctx, const InspectionBuffer *buffer = HttpRequestBodyGetDataCallback( det_ctx, engine->v2.transforms, f, flags, txv, engine->sm_list, engine->sm_list_base); if (buffer == NULL || buffer->inspect == NULL) { + if (eof && engine->match_on_null) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } return eof ? DETECT_ENGINE_INSPECT_SIG_CANT_MATCH : DETECT_ENGINE_INSPECT_SIG_NO_MATCH; } diff --git a/src/detect-http-header.c b/src/detect-http-header.c index 0a1a8ecadeed..22a487583ae7 100644 --- a/src/detect-http-header.c +++ b/src/detect-http-header.c @@ -191,6 +191,9 @@ static uint8_t DetectEngineInspectBufferHttpHeader(DetectEngineCtx *de_ctx, uint8_t *rawdata = GetBufferForTX(txv, det_ctx, f, flags, &rawdata_len); if (rawdata_len == 0) { SCLogDebug("no data"); + if (engine->match_on_null && eof) { + return DETECT_ENGINE_INSPECT_SIG_MATCH; + } goto end; } /* setup buffer and apply transforms */ diff --git a/src/detect-isdataat.c b/src/detect-isdataat.c index 7b4d629ad3a1..6bc4e961c07a 100644 --- a/src/detect-isdataat.c +++ b/src/detect-isdataat.c @@ -35,6 +35,7 @@ #include "detect-isdataat.h" #include "detect-content.h" +#include "detect-bytetest.h" #include "detect-uricontent.h" #include "detect-engine-build.h" @@ -56,11 +57,99 @@ static DetectParseRegex parse_regex; int DetectIsdataatSetup (DetectEngineCtx *, Signature *, const char *); #ifdef UNITTESTS static void DetectIsdataatRegisterTests(void); +static void DetectAbsentRegisterTests(void); #endif void DetectIsdataatFree(DetectEngineCtx *, void *); static int DetectEndsWithSetup (DetectEngineCtx *de_ctx, Signature *s, const char *nullstr); +static void DetectAbsentFree(DetectEngineCtx *de_ctx, void *ptr) +{ + SCFree(ptr); +} + +static int DetectAbsentSetup(DetectEngineCtx *de_ctx, Signature *s, const char *optstr) +{ + if (s->init_data->list == DETECT_SM_LIST_NOTSET) { + SCLogError("no buffer for absent keyword"); + return -1; + } + + if (DetectBufferGetActiveList(de_ctx, s) == -1) + return -1; + + bool or_else; + if (optstr == NULL) { + or_else = false; + } else if (strcmp(optstr, "or_else") == 0) { + or_else = true; + } else { + SCLogError("unhandled value for absent keyword: %s", optstr); + return -1; + } + if (s->init_data->curbuf == NULL || s->init_data->list != (int)s->init_data->curbuf->id) { + SCLogError("unspected buffer for absent keyword"); + return -1; + } + const DetectBufferType *b = DetectEngineBufferTypeGetById(de_ctx, s->init_data->list); + if (!b || b->frame) { + SCLogError("absent does not work with frames"); + return -1; + } + if (s->init_data->curbuf->tail != NULL) { + SCLogError("absent must come first right after buffer"); + return -1; + } + DetectAbsentData *dad = SCMalloc(sizeof(DetectAbsentData)); + if (unlikely(dad == NULL)) + return -1; + + dad->or_else = or_else; + + if (SigMatchAppendSMToList(de_ctx, s, DETECT_ABSENT, (SigMatchCtx *)dad, s->init_data->list) == + NULL) { + DetectAbsentFree(de_ctx, dad); + return -1; + } + return 0; +} + +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *b) +{ + bool has_other = false; + bool only_absent = false; + bool has_absent = false; + for (const SigMatch *sm = b->head; sm != NULL; sm = sm->next) { + if (sm->type == DETECT_ABSENT) { + has_absent = true; + const DetectAbsentData *dad = (const DetectAbsentData *)sm->ctx; + if (!dad->or_else) { + only_absent = true; + } + } else { + has_other = true; + if (sm->type == DETECT_CONTENT) { + const DetectContentData *cd = (DetectContentData *)sm->ctx; + if (has_absent && (cd->flags & DETECT_CONTENT_FAST_PATTERN)) { + SCLogError("signature can't have absent and fast_pattern on the same buffer"); + return false; + } + } + } + } + + if (only_absent && has_other) { + SCLogError("signature can't have a buffer tested absent and tested with other keywords " + "such as content"); + return false; + } else if (has_absent && !only_absent && !has_other) { + SCLogError( + "signature with absent: or_else expects other keywords to test on such as content"); + return false; + } + return true; +} + /** * \brief Registration function for isdataat: keyword */ @@ -82,6 +171,16 @@ void DetectIsdataatRegister(void) sigmatch_table[DETECT_ENDS_WITH].Setup = DetectEndsWithSetup; sigmatch_table[DETECT_ENDS_WITH].flags = SIGMATCH_NOOPT; + sigmatch_table[DETECT_ABSENT].name = "absent"; + sigmatch_table[DETECT_ABSENT].desc = "test if the buffer is absent"; + sigmatch_table[DETECT_ABSENT].url = "/rules/payload-keywords.html#absent"; + sigmatch_table[DETECT_ABSENT].Setup = DetectAbsentSetup; + sigmatch_table[DETECT_ABSENT].Free = DetectAbsentFree; + sigmatch_table[DETECT_ABSENT].flags = SIGMATCH_OPTIONAL_OPT; +#ifdef UNITTESTS + sigmatch_table[DETECT_ABSENT].RegisterTests = DetectAbsentRegisterTests; +#endif + DetectSetupParseRegexes(PARSE_REGEX, &parse_regex); } @@ -584,4 +683,41 @@ void DetectIsdataatRegisterTests(void) UtRegisterTest("DetectIsdataatTestPacket02", DetectIsdataatTestPacket02); UtRegisterTest("DetectIsdataatTestPacket03", DetectIsdataatTestPacket03); } + +static int DetectAbsentTestParse01(void) +{ + DetectEngineCtx *de_ctx = DetectEngineCtxInit(); + FAIL_IF(de_ctx == NULL); + de_ctx->flags |= DE_QUIET; + + Signature *s = DetectEngineAppendSig(de_ctx, + "alert http any any -> any any " + "(msg:\"invalid absent only with negated content\"; http.user_agent; " + "absent; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:!\"one\"; absent; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"invalid absent\"; http.user_agent; " + "content:\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, "alert http any any -> any any " + "(msg:\"absent without sticky buffer\"; " + "content:!\"one\"; absent: or_else; sid:2;)"); + FAIL_IF(s != NULL); + s = DetectEngineAppendSig(de_ctx, + "alert websocket any any -> any any " + "(msg:\"absent with frame\"; " + "frame: websocket.pdu; absent: or_else; content:!\"one\"; sid:2;)"); + FAIL_IF(s != NULL); + DetectEngineCtxFree(de_ctx); + PASS; +} + +void DetectAbsentRegisterTests(void) +{ + UtRegisterTest("DetectAbsentTestParse01", DetectAbsentTestParse01); +} #endif diff --git a/src/detect-isdataat.h b/src/detect-isdataat.h index 01ea2e304f42..c0cff8848385 100644 --- a/src/detect-isdataat.h +++ b/src/detect-isdataat.h @@ -34,7 +34,14 @@ typedef struct DetectIsdataatData_ { uint8_t flags; /* isdataat options*/ } DetectIsdataatData; +typedef struct DetectAbsentData_ { + /** absent or try to match with other keywords (false means only absent) */ + bool or_else; +} DetectAbsentData; + /* prototypes */ void DetectIsdataatRegister (void); +bool DetectAbsentValidateContentCallback(Signature *s, const SignatureInitDataBuffer *); + #endif /* SURICATA_DETECT_ISDATAAT_H */ diff --git a/src/detect-parse.c b/src/detect-parse.c index 984501c1dd8a..3b03dfb92b36 100644 --- a/src/detect-parse.c +++ b/src/detect-parse.c @@ -35,6 +35,7 @@ #include "detect-content.h" #include "detect-bsize.h" +#include "detect-isdataat.h" #include "detect-pcre.h" #include "detect-uricontent.h" #include "detect-reference.h" @@ -1971,6 +1972,9 @@ static int SigValidate(DetectEngineCtx *de_ctx, Signature *s) if (!DetectBsizeValidateContentCallback(s, b)) { SCReturnInt(0); } + if (!DetectAbsentValidateContentCallback(s, b)) { + SCReturnInt(0); + } } int ts_excl = 0; diff --git a/src/detect.h b/src/detect.h index 4c22f509a3f3..e88b5540ac79 100644 --- a/src/detect.h +++ b/src/detect.h @@ -432,6 +432,8 @@ typedef struct DetectEngineAppInspectionEngine_ { uint8_t id; /**< per sig id used in state keeping */ bool mpm; bool stream; + /** will match on a NULL buffer (so an absent buffer) */ + bool match_on_null; uint16_t sm_list; uint16_t sm_list_base; /**< base buffer being transformed */ int16_t progress;