diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..503ca3d --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +*.o +*.so +*.a +*.la +*.exe +*.dll +*.swp +*~ + diff --git a/Makefile.am b/Makefile.am index 2b0ca10..b276381 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4,7 +4,7 @@ AUTOMAKE_OPTIONS=foreign no-dependencies libdir = ${exec_prefix}/lib/snort_dynamicpreprocessor lib_LTLIBRARIES = libsf_ai_preproc.la -libsf_ai_preproc_la_CFLAGS = -I./uthash -I./base64 -I./fsom -I./include ${LIBXML2_INCLUDES} ${LIBGRAPH_INCLUDES} -DDYNAMIC_PLUGIN -D_XOPEN_SOURCE -D_GNU_SOURCE -fvisibility=hidden -fno-strict-aliasing -Wall -pedantic -pedantic-errors -std=c99 -fstack-protector +libsf_ai_preproc_la_CFLAGS = -I./uthash -I./base64 -I./fsom -I./include ${LIBXML2_INCLUDES} ${LIBGRAPH_INCLUDES} ${LIBPYTHON_INCLUDES} -DDYNAMIC_PLUGIN -D_XOPEN_SOURCE -D_GNU_SOURCE -fvisibility=hidden -fno-strict-aliasing -Wall -pedantic -pedantic-errors -std=c99 -fstack-protector libsf_ai_preproc_la_LDFLAGS = -module -export-dynamic BUILT_SOURCES = \ @@ -28,6 +28,8 @@ db.c \ fkmeans/kmeans.c \ fsom/fsom.c \ geo.c \ +kb.c \ +manual.c \ modules.c \ mysql.c \ neural.c \ diff --git a/Makefile.in b/Makefile.in index 93b3ffb..7956509 100644 --- a/Makefile.in +++ b/Makefile.in @@ -85,7 +85,8 @@ am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_history.lo \ libsf_ai_preproc_la-cluster.lo \ libsf_ai_preproc_la-correlation.lo libsf_ai_preproc_la-db.lo \ libsf_ai_preproc_la-kmeans.lo libsf_ai_preproc_la-fsom.lo \ - libsf_ai_preproc_la-geo.lo libsf_ai_preproc_la-modules.lo \ + libsf_ai_preproc_la-geo.lo libsf_ai_preproc_la-kb.lo \ + libsf_ai_preproc_la-manual.lo libsf_ai_preproc_la-modules.lo \ libsf_ai_preproc_la-mysql.lo libsf_ai_preproc_la-neural.lo \ libsf_ai_preproc_la-neural_cluster.lo \ libsf_ai_preproc_la-outdb.lo libsf_ai_preproc_la-postgresql.lo \ @@ -165,6 +166,7 @@ LD = @LD@ LDFLAGS = @LDFLAGS@ LIBGRAPH_INCLUDES = @LIBGRAPH_INCLUDES@ LIBOBJS = @LIBOBJS@ +LIBPYTHON_INCLUDES = @LIBPYTHON_INCLUDES@ LIBS = @LIBS@ LIBTOOL = @LIBTOOL@ LIBXML2_INCLUDES = @LIBXML2_INCLUDES@ @@ -249,7 +251,7 @@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ AUTOMAKE_OPTIONS = foreign no-dependencies lib_LTLIBRARIES = libsf_ai_preproc.la -libsf_ai_preproc_la_CFLAGS = -I./uthash -I./base64 -I./fsom -I./include ${LIBXML2_INCLUDES} ${LIBGRAPH_INCLUDES} -DDYNAMIC_PLUGIN -D_XOPEN_SOURCE -D_GNU_SOURCE -fvisibility=hidden -fno-strict-aliasing -Wall -pedantic -pedantic-errors -std=c99 -fstack-protector +libsf_ai_preproc_la_CFLAGS = -I./uthash -I./base64 -I./fsom -I./include ${LIBXML2_INCLUDES} ${LIBGRAPH_INCLUDES} ${LIBPYTHON_INCLUDES} -DDYNAMIC_PLUGIN -D_XOPEN_SOURCE -D_GNU_SOURCE -fvisibility=hidden -fno-strict-aliasing -Wall -pedantic -pedantic-errors -std=c99 -fstack-protector libsf_ai_preproc_la_LDFLAGS = -module -export-dynamic BUILT_SOURCES = \ include/sf_dynamic_preproc_lib.c \ @@ -272,6 +274,8 @@ db.c \ fkmeans/kmeans.c \ fsom/fsom.c \ geo.c \ +kb.c \ +manual.c \ modules.c \ mysql.c \ neural.c \ @@ -430,6 +434,12 @@ libsf_ai_preproc_la-fsom.lo: fsom/fsom.c libsf_ai_preproc_la-geo.lo: geo.c $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-geo.lo `test -f 'geo.c' || echo '$(srcdir)/'`geo.c +libsf_ai_preproc_la-kb.lo: kb.c + $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-kb.lo `test -f 'kb.c' || echo '$(srcdir)/'`kb.c + +libsf_ai_preproc_la-manual.lo: manual.c + $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-manual.lo `test -f 'manual.c' || echo '$(srcdir)/'`manual.c + libsf_ai_preproc_la-modules.lo: modules.c $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-modules.lo `test -f 'modules.c' || echo '$(srcdir)/'`modules.c diff --git a/README b/README index 61468fd..e2bae56 100644 --- a/README +++ b/README @@ -195,6 +195,7 @@ preprocessor ai: \ output_database ( type="dbtype", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \ output_neurons_per_side 20 \ tcp_stream_expire_interval 300 \ + use_knowledge_base_correlation_index 1 \ use_stream_hash_table 1 \ webserv_banner "Snort AIPreprocessor module" \ webserv_dir "/prefix/share/htdocs" \ @@ -378,19 +379,26 @@ stream as "expired", if no more packets are received inside of that and it's not "marked" as suspicious (default if not specified: 300 seconds) -- use_stream_hash_table: Set this option to 0 if you do not want to use the hash -table for storing the streams of packets associated to alerts, this is a good -choice on a system where many alerts are triggered (default value if not +- use_knowledge_base_correlation_index: Set this option to 0 if you do not want +to use the knowledge base alert correlation index (default value if not +specified: 1) + + +- use_stream_hash_table: Set this option to 0 if you do not want to use the +hash table for storing the streams of packets associated to alerts, this is a +good choice on a system where many alerts are triggered (default value if not specified: 1) - webserv_banner: Banner of the web server, to be placed on the error pages and in the "Server" HTTP reply header + - webserver_dir: Directory containing the contents of the web server running over the module (default if none is specified: $PREFIX/share/snort_ai_preprocessor/htdocs) + - webserver_port: Port where the web server will listen (default if none is specified: 7654). Set this value to 0 if you don't want to run the web server over the module for having the web interface (in this case, if you want to see diff --git a/config.h.in b/config.h.in index 5b3be33..237ed5e 100644 --- a/config.h.in +++ b/config.h.in @@ -81,6 +81,9 @@ /* Define to 1 if you have the `pthread' library (-lpthread). */ #undef HAVE_LIBPTHREAD +/* Define to 1 if you have the `python2.6' library (-lpython2.6). */ +#undef HAVE_LIBPYTHON2_6 + /* Define to 1 if you have the `xml2' library (-lxml2). */ #undef HAVE_LIBXML2 diff --git a/configure b/configure index 6dbb826..1632cab 100755 --- a/configure +++ b/configure @@ -752,6 +752,7 @@ LTLIBOBJS LIBOBJS ALLOCA LIBGRAPH_INCLUDES +LIBPYTHON_INCLUDES LIBXML2_INCLUDES SHARE_PREFIX DOC_PREFIX @@ -874,6 +875,7 @@ with_gnu_ld enable_libtool_lock with_mysql with_postgresql +with_python with_graphviz ' ac_precious_vars='build_alias @@ -1526,6 +1528,7 @@ Optional Packages: --with-postgresql Enable support for PostgreSQL alert logs [default=no] WARNING: You cannot enable the support for two databases at the same time + --with-python Enable support for Python modules [default=no] --without-graphviz Disable Graphviz support for rendering correlated alerts as a PNG graph [default=yes] @@ -4593,13 +4596,13 @@ if test "${lt_cv_nm_interface+set}" = set; then : else lt_cv_nm_interface="BSD nm" echo "int some_variable = 0;" > conftest.$ac_ext - (eval echo "\"\$as_me:4596: $ac_compile\"" >&5) + (eval echo "\"\$as_me:4599: $ac_compile\"" >&5) (eval "$ac_compile" 2>conftest.err) cat conftest.err >&5 - (eval echo "\"\$as_me:4599: $NM \\\"conftest.$ac_objext\\\"\"" >&5) + (eval echo "\"\$as_me:4602: $NM \\\"conftest.$ac_objext\\\"\"" >&5) (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out) cat conftest.err >&5 - (eval echo "\"\$as_me:4602: output\"" >&5) + (eval echo "\"\$as_me:4605: output\"" >&5) cat conftest.out >&5 if $GREP 'External.*some_variable' conftest.out > /dev/null; then lt_cv_nm_interface="MS dumpbin" @@ -5805,7 +5808,7 @@ ia64-*-hpux*) ;; *-*-irix6*) # Find out which ABI we are using. - echo '#line 5808 "configure"' > conftest.$ac_ext + echo '#line 5811 "configure"' > conftest.$ac_ext if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5 (eval $ac_compile) 2>&5 ac_status=$? @@ -7334,11 +7337,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7337: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7340: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7341: \$? = $ac_status" >&5 + echo "$as_me:7344: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7673,11 +7676,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7676: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7679: $lt_compile\"" >&5) (eval "$lt_compile" 2>conftest.err) ac_status=$? cat conftest.err >&5 - echo "$as_me:7680: \$? = $ac_status" >&5 + echo "$as_me:7683: \$? = $ac_status" >&5 if (exit $ac_status) && test -s "$ac_outfile"; then # The compiler can only warn and ignore the option if not recognized # So say no if there are warnings other than the usual output. @@ -7778,11 +7781,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7781: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7784: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7785: \$? = $ac_status" >&5 + echo "$as_me:7788: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -7833,11 +7836,11 @@ else -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \ -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \ -e 's:$: $lt_compiler_flag:'` - (eval echo "\"\$as_me:7836: $lt_compile\"" >&5) + (eval echo "\"\$as_me:7839: $lt_compile\"" >&5) (eval "$lt_compile" 2>out/conftest.err) ac_status=$? cat out/conftest.err >&5 - echo "$as_me:7840: \$? = $ac_status" >&5 + echo "$as_me:7843: \$? = $ac_status" >&5 if (exit $ac_status) && test -s out/conftest2.$ac_objext then # The compiler can only warn and ignore the option if not recognized @@ -10217,7 +10220,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 10220 "configure" +#line 10223 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -10313,7 +10316,7 @@ else lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2 lt_status=$lt_dlunknown cat > conftest.$ac_ext <<_LT_EOF -#line 10316 "configure" +#line 10319 "configure" #include "confdefs.h" #if HAVE_DLFCN_H @@ -11394,6 +11397,15 @@ fi +# Check whether --with-python was given. +if test "${with_python+set}" = set; then : + withval=$with_python; with_python=yes +else + with_python=no +fi + + + # Check whether --with-graphviz was given. if test "${with_graphviz+set}" = set; then : withval=$with_graphviz; @@ -11509,6 +11521,59 @@ fi fi +if test "x$with_python" != xno; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Py_BuildValue in -lpython2.6" >&5 +$as_echo_n "checking for Py_BuildValue in -lpython2.6... " >&6; } +if test "${ac_cv_lib_python2_6_Py_BuildValue+set}" = set; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-lpython2.6 $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char Py_BuildValue (); +int +main () +{ +return Py_BuildValue (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_python2_6_Py_BuildValue=yes +else + ac_cv_lib_python2_6_Py_BuildValue=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_python2_6_Py_BuildValue" >&5 +$as_echo "$ac_cv_lib_python2_6_Py_BuildValue" >&6; } +if test "x$ac_cv_lib_python2_6_Py_BuildValue" = x""yes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBPYTHON2_6 1 +_ACEOF + + LIBS="-lpython2.6 $LIBS" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "--with-python option used, but libpython2.6 was not found - do not use --with-python, or, on a Debian-based system, install libpython2.6 +See \`config.log' for more details" "$LINENO" 5 ; } +fi + +fi + if test "x$with_mysql" != xno -a "x$with_postgresql" != xno; then : { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} @@ -11799,6 +11864,18 @@ as_fn_error $? "libxml2 not found, okr pkg-config not working See \`config.log' for more details" "$LINENO" 5 ; } fi +if test "x$with_python" != xno; then : + if test ! -z "`ls /usr/include/python2.6 2> /dev/null`"; then : + LIBPYTHON_INCLUDES="-I/usr/include/python2.6" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "python support enabled, but the include directory /usr/include/python2.6 was not found +See \`config.log' for more details" "$LINENO" 5 ; } +fi +fi + if test "x$with_graphviz" != xno; then : if test ! -z "`pkg-config --cflags libgraph 2> /dev/null`"; then : LIBGRAPH_INCLUDES="$(pkg-config --cflags libgraph 2> /dev/null)" diff --git a/configure.ac b/configure.ac index 7990642..9e1d695 100644 --- a/configure.ac +++ b/configure.ac @@ -111,6 +111,12 @@ AC_ARG_WITH(postgresql, [with_postgresql=yes], [with_postgresql=no]) +AC_ARG_WITH(python, + AS_HELP_STRING([--with-python], + [Enable support for Python modules @<:@default=no@:>@]), + [with_python=yes], + [with_python=no]) + AC_ARG_WITH(graphviz, AS_HELP_STRING([--without-graphviz], [Disable Graphviz support for rendering correlated alerts as a PNG graph @<:@default=yes@:>@]), @@ -126,6 +132,10 @@ AS_IF([test "x$with_postgresql" != xno], [AC_CHECK_LIB([pq], [PQexec],, [AC_MSG_FAILURE([--with-postgresql option used, but libpq was not found - do not use --with-postgresql, or, on a Debian-based system, install libpq-dev])])]) +AS_IF([test "x$with_python" != xno], + [AC_CHECK_LIB([python2.6], [Py_BuildValue],, + [AC_MSG_FAILURE([--with-python option used, but libpython2.6 was not found - do not use --with-python, or, on a Debian-based system, install libpython2.6])])]) + AS_IF([test "x$with_mysql" != xno -a "x$with_postgresql" != xno], [AC_MSG_FAILURE([The support for an only DBMS can be enabled via ./configure, for example, you cannot enable MySQL and PostgreSQL support at the same time])], []) @@ -153,6 +163,11 @@ AS_IF([test ! -z "`pkg-config --cflags libxml-2.0 2> /dev/null`"], [AC_SUBST([LIBXML2_INCLUDES], ["$(pkg-config --cflags libxml-2.0 2> /dev/null)"])], [AC_MSG_FAILURE([libxml2 not found, okr pkg-config not working])]) +AS_IF([test "x$with_python" != xno], + [AS_IF([test ! -z "`ls /usr/include/python2.6 2> /dev/null`"], + [AC_SUBST([LIBPYTHON_INCLUDES], ["-I/usr/include/python2.6"])], + [AC_MSG_FAILURE([python support enabled, but the include directory /usr/include/python2.6 was not found])])]) + AS_IF([test "x$with_graphviz" != xno], [AS_IF([test ! -z "`pkg-config --cflags libgraph 2> /dev/null`"], [AC_SUBST([LIBGRAPH_INCLUDES], ["$(pkg-config --cflags libgraph 2> /dev/null)"])], diff --git a/correlation.c b/correlation.c index d1e09d0..36e984f 100644 --- a/correlation.c +++ b/correlation.c @@ -20,7 +20,6 @@ #include "spp_ai.h" #include -#include #include #include #include @@ -33,39 +32,11 @@ #include #endif -/** \defgroup correlation Module for the correlation of hyperalerts +/** \defgroup correlation Module for the correlation of security alerts * @{ */ -#ifndef LIBXML_READER_ENABLED -#error "libxml2 reader not enabled\n" -#endif - -/** Enumeration for the types of hyperalert XML tags */ -enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, HYP_TAG_NUM }; - -/** Enumeration for the types of manual correlations XML tags */ -enum { inCorrelation, inCorrelations, inFromTag, inToTag, MAN_TAG_NUM }; - -typedef struct { - int from_gid; - int from_sid; - int from_rev; - int to_gid; - int to_sid; - int to_rev; -} AI_alert_type_pair_key; - -typedef struct { - AI_alert_type_pair_key key; - enum { manuallyNone, manuallyCorrelated, manuallyNotCorrelated } corr_type; - UT_hash_handle hh; -} AI_alert_type_pair; - -PRIVATE AI_hyperalert_info *hyperalerts = NULL; PRIVATE AI_snort_alert *alerts = NULL; PRIVATE AI_alert_correlation *correlation_table = NULL; -PRIVATE AI_alert_type_pair *manual_correlations = NULL; -PRIVATE AI_alert_type_pair *manual_uncorrelations = NULL; PRIVATE pthread_mutex_t mutex; /** @@ -344,866 +315,6 @@ __AI_correlated_alerts_to_json () chmod ( json_file, 0644 ); } /* ----- end of function __AI_correlated_alerts_to_json ----- */ -/** - * \brief Get the name of the function called by a pre-condition or post-condition predicate - * \param orig_stmt Statement representing a pre-condition or post-condition - * \return The name of the function called by that statement - */ - -PRIVATE char* -__AI_get_function_name ( const char *orig_stmt ) -{ - int parenthesis_pos, function_name_len; - char function_name[4096]; - char *stmt = NULL; - - if ( !( stmt = (char*) alloca ( strlen ( orig_stmt )))) - return NULL; - strcpy ( stmt, orig_stmt ); - - memset ( function_name, 0, sizeof ( function_name )); - - if ( !( parenthesis_pos = (int) strstr ( stmt, "(" ))) - return NULL; - - parenthesis_pos -= (int) stmt; - function_name_len = ( parenthesis_pos < sizeof ( function_name )) ? parenthesis_pos : sizeof ( function_name ); - strncpy ( function_name, stmt, function_name_len ); - - return strdup(function_name); -} /* ----- end of function __AI_get_function_name ----- */ - - -/** - * \brief Get the arguments passed to a function predicate in a pre-condition or post-condition (comma-separated values) - * \param orig_stmt Statement representing a pre-condition or post-condition - * \param n_args Reference to an integer that will contain the number of arguments read - * \return An array of strings containing the arguments of the function - */ - -PRIVATE char** -__AI_get_function_arguments ( char *orig_stmt, int *n_args ) -{ - char **args = NULL; - char *tok = NULL; - char *stmt = NULL; - int par_pos = 0; - *n_args = 0; - - if ( !( stmt = (char*) alloca ( strlen ( orig_stmt )))) - return NULL; - strcpy ( stmt, orig_stmt ); - - if ( !( par_pos = (int) strstr ( stmt, "(" ))) - return NULL; - - par_pos -= (int) stmt; - stmt += par_pos + 1; - - if ( stmt [ strlen(stmt) - 1 ] == ')' ) - stmt[ strlen(stmt) - 1 ] = 0; - - tok = (char*) strtok ( stmt, "," ); - - while ( tok ) { - if ( !( args = (char**) realloc ( args, (++(*n_args)) * sizeof ( char* )))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - args [ (*n_args) - 1 ] = strdup ( tok ); - tok = (char*) strtok ( NULL, " " ); - } - - if ( !(*n_args) ) - return NULL; - - return args; -} /* ----- end of function __AI_get_function_arguments ----- */ - - -/** - * \brief Compute the correlation coefficient between two alerts, as #INTERSECTION(pre(B), post(A)) / #UNION(pre(B), post(A)), on the basis of preconditions and postconditions in the knowledge base's correlation rules - * \param a Alert a - * \param b Alert b - * \return The correlation coefficient between A and B as coefficient in [0,1] - */ - -PRIVATE double -__AI_kb_correlation_coefficient ( AI_snort_alert *a, AI_snort_alert *b ) -{ - unsigned int i, j, k, l, - n_intersection = 0, - n_union = 0; - - char **args1 = NULL, - **args2 = NULL, - **matches = NULL, - *function_name1 = NULL, - *function_name2 = NULL, - new_stmt1[4096] = {0}, - new_stmt2[4096] = {0}; - - int n_args1 = 0, - n_args2 = 0, - n_matches = 0, - min_addr = 0, - max_addr = 0, - ipaddr = 0, - netmask = 0; - - if ( !a->hyperalert || !b->hyperalert ) - return 0.0; - - if ( a->hyperalert->n_postconds == 0 || b->hyperalert->n_preconds == 0 ) - return 0.0; - - n_union = a->hyperalert->n_postconds + b->hyperalert->n_preconds; - - for ( i=0; i < a->hyperalert->n_postconds; i++ ) - { - for ( j=0; j < b->hyperalert->n_preconds; j++ ) - { - if ( !strcasecmp ( a->hyperalert->postconds[i], b->hyperalert->preconds[j] )) - { - n_intersection += 2; - } else { - /* Check if the predicates are the same, have the same number of arguments, and - * substitute possible occurrencies of +ANY_ADDR+ and +ANY_PORT+ or IP netmasks */ - function_name1 = __AI_get_function_name ( a->hyperalert->postconds[i] ); - function_name2 = __AI_get_function_name ( b->hyperalert->preconds[j] ); - - if ( !strcasecmp ( function_name1, function_name2 )) - { - args1 = __AI_get_function_arguments ( a->hyperalert->postconds[i], &n_args1 ); - args2 = __AI_get_function_arguments ( b->hyperalert->preconds[j] , &n_args2 ); - - if ( args1 && args2 ) - { - if ( n_args1 == n_args2 ) - { - memset ( new_stmt1, 0, sizeof ( new_stmt1 )); - memset ( new_stmt2, 0, sizeof ( new_stmt2 )); - - for ( k=0; k < n_args1; k++ ) - { - /* If any occurrence of +ANY_ADDR+ or +ANY_PORT+ is found in any of the arguments, - * substitute that occurrence with the associated value */ - if ( !strcasecmp ( args1[k], "+ANY_ADDR+" ) || !strcasecmp ( args1[k], "+ANY_PORT+" )) - { - free ( args1[k] ); - args1[k] = args2[k]; - } - - if ( !strcasecmp ( args2[k], "+ANY_ADDR+" ) || !strcasecmp ( args2[k], "+ANY_PORT+" )) - { - free ( args2[k] ); - args2[k] = args1[k]; - } - - /* Substitute any occurrence of an IP netmask in any of the two arguments with - * the associated IP value */ - if ( preg_match ( "^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})/([0-9]{1,2})$", args1[k], &matches, &n_matches )) - { - if ( preg_match ( "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", args2[k], NULL, NULL )) - { - if (( netmask = strtol ( matches[1], NULL, 10 )) > 32 ) - AI_fatal_err ( "Invalid IP netmask value in configuration", __FILE__, __LINE__ ); - - if (( min_addr = inet_addr ( matches[0] )) == INADDR_NONE ) - AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); - - ipaddr = inet_addr ( args2[k] ); - - if ( ipaddr == INADDR_NONE ) - AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); - - netmask = 1 << (( 8*sizeof ( uint32_t )) - netmask ); - min_addr = ntohl ( min_addr ) & (~(netmask - 1)); - max_addr = min_addr | (netmask - 1); - ipaddr = ntohl ( ipaddr ); - - if ( ipaddr >= min_addr && ipaddr <= max_addr ) - { - free ( args1[k] ); - args1[k] = args2[k]; - } - } - - for ( l=0; l < n_matches; l++ ) - free ( matches[l] ); - free ( matches ); - } - - if ( preg_match ( "^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})/([0-9]{1,2})$", args2[k], &matches, &n_matches )) - { - if ( preg_match ( "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", args1[k], NULL, NULL )) - { - if (( netmask = strtol ( matches[1], NULL, 10 )) > 32 ) - AI_fatal_err ( "Invalid IP netmask value in configuration", __FILE__, __LINE__ ); - - if (( min_addr = inet_addr ( matches[0] )) == INADDR_NONE ) - AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); - - ipaddr = inet_addr ( args1[k] ); - - if ( ipaddr == INADDR_NONE ) - AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); - - netmask = 1 << (( 8*sizeof ( uint32_t )) - netmask ); - min_addr = ntohl ( min_addr ) & (~(netmask - 1)); - max_addr = min_addr | (netmask - 1); - ipaddr = ntohl ( ipaddr ); - - if ( ipaddr >= min_addr && ipaddr <= max_addr ) - { - free ( args2[k] ); - args2[k] = args1[k]; - } - } - - for ( l=0; l < n_matches; l++ ) - free ( matches[l] ); - free ( matches ); - } - } - - snprintf ( new_stmt1, sizeof ( new_stmt1 ), "%s(", function_name1 ); - snprintf ( new_stmt2, sizeof ( new_stmt2 ), "%s(", function_name2 ); - - for ( k=0; k < n_args1; k++ ) - { - if ( strlen ( new_stmt1 ) + strlen ( args1[k] ) + 1 < sizeof ( new_stmt1 )) - sprintf ( new_stmt1, "%s%s%s", new_stmt1, args1[k], ( k < n_args1 - 1 ) ? "," : ")" ); - - if ( strlen ( new_stmt2 ) + strlen ( args2[k] ) + 1 < sizeof ( new_stmt2 )) - sprintf ( new_stmt2, "%s%s%s", new_stmt2, args2[k], ( k < n_args2 - 1 ) ? "," : ")" ); - } - - if ( !strcmp ( new_stmt1, new_stmt2 )) - { - n_intersection += 2; - } - } - - for ( k=0; k < n_args1; k++ ) - { - if ( args1[k] ) - { - free ( args1[k] ); - args1[k] = NULL; - } - } - - if ( args1 ) - { - free ( args1 ); - args1 = NULL; - } - - for ( k=0; k < n_args2; k++ ) - { - if ( args2[k] ) - { - /* free ( args2[k] ); */ - args2[k] = NULL; - } - } - - if ( args2 ) - { - free ( args2 ); - args2 = NULL; - } - } - } - - if ( function_name1 ) - { - free ( function_name1 ); - function_name1 = NULL; - } - - if ( function_name2 ) - { - free ( function_name2 ); - function_name2 = NULL; - } - } - } - } - - return (double) ((double) n_intersection / (double) n_union ); -} /* ----- end of function __AI_kb_correlation_coefficient ----- */ - - -/** - * \brief Parse the manual specified correlations from XML file(s) and fills the hash table - */ - -PRIVATE void* -__AI_manual_correlations_parsing_thread ( void *arg ) -{ - unsigned int i = 0; - char manual_correlations_xml[1060] = { 0 }, - manual_uncorrelations_xml[1060] = { 0 }; - struct stat st; - xmlTextReaderPtr xml; - const xmlChar *tagname; - AI_alert_type_pair_key key; - AI_alert_type_pair *pair = NULL, - *found = NULL; - BOOL xml_flags[MAN_TAG_NUM] = { false }; - - while ( 1 ) - { - /* Cleanup tables */ - while ( manual_correlations ) - { - pair = manual_correlations; - HASH_DEL ( manual_correlations, pair ); - free ( pair ); - } - - while ( manual_uncorrelations ) - { - pair = manual_uncorrelations; - HASH_DEL ( manual_uncorrelations, pair ); - free ( pair ); - } - - pair = NULL; - memset ( &key, 0, sizeof ( key )); - - snprintf ( manual_correlations_xml, - sizeof ( manual_correlations_xml ), - "%s/manual_correlations.xml", config->webserv_dir ); - - snprintf ( manual_uncorrelations_xml, - sizeof ( manual_uncorrelations_xml ), - "%s/manual_uncorrelations.xml", config->webserv_dir ); - - if ( stat ( manual_correlations_xml, &st ) < 0 ) - { - pthread_exit ((void*) 0); - return (void*) 0; - } - - if ( stat ( manual_uncorrelations_xml, &st ) < 0 ) - { - pthread_exit ((void*) 0); - return (void*) 0; - } - - LIBXML_TEST_VERSION - - /* Check manual correlations */ - if ( !( xml = xmlReaderForFile ( manual_correlations_xml, NULL, 0 ))) - { - pthread_exit ((void*) 0); - return (void*) 0; - } - - while ( xmlTextReaderRead ( xml )) - { - if ( !( tagname = xmlTextReaderConstName ( xml ))) - continue; - - if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) - { - if ( !strcasecmp ((const char*) tagname, "correlations" )) - { - if ( xml_flags[inCorrelations] ) - { - AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelations] = true; - } - } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { - if ( xml_flags[inCorrelation] ) - { - AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelation] = true; - } - } else if ( !strcasecmp ((const char*) tagname, "from" )) { - xml_flags[inFromTag] = true; - - key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; - key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; - key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; - - /* If this is a new pair, allocate the memory */ - if ( pair == NULL ) - { - if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) - { - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - } - - pair->corr_type = manuallyCorrelated; - } else { - /* Otherwise, add the pair to the hash, if it's not already there */ - pair->key = key; - HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found ); - - if ( !found ) - { - HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair ); - } - - pair = NULL; - memset ( &key, 0, sizeof ( key )); - } - } else if ( !strcasecmp ((const char*) tagname, "to" )) { - xml_flags[inToTag] = true; - - key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; - key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; - key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; - - /* If this is a new pair, allocate the memory */ - if ( pair == NULL ) - { - if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) - { - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - } - - pair->corr_type = manuallyCorrelated; - } else { - /* Otherwise, add the pair to the hash, if it's not already there */ - pair->key = key; - HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found ); - - if ( !found ) - { - HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair ); - } - - pair = NULL; - memset ( &key, 0, sizeof ( key )); - } - } else { - AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); - } - } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { - if ( !strcasecmp ((const char*) tagname, "correlations" )) - { - if ( !xml_flags[inCorrelations] ) - { - AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelations] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { - if ( !xml_flags[inCorrelation] ) - { - AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelation] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "from" )) { - if ( !xml_flags[inFromTag] ) - { - AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inFromTag] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "to" )) { - if ( !xml_flags[inToTag] ) - { - AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inToTag] = false; - } - } else { - AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); - } - } - } - - xmlFreeTextReader ( xml ); - xmlCleanupParser(); - - for ( i=0; i < MAN_TAG_NUM; i++ ) - { - xml_flags[i] = false; - } - - /* Check manual un-correlations */ - if ( !( xml = xmlReaderForFile ( manual_uncorrelations_xml, NULL, 0 ))) - { - pthread_exit ((void*) 0); - return (void*) 0; - } - - while ( xmlTextReaderRead ( xml )) - { - if ( !( tagname = xmlTextReaderConstName ( xml ))) - continue; - - if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) - { - if ( !strcasecmp ((const char*) tagname, "correlations" )) - { - if ( xml_flags[inCorrelations] ) - { - AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelations] = true; - } - } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { - if ( xml_flags[inCorrelation] ) - { - AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelation] = true; - } - } else if ( !strcasecmp ((const char*) tagname, "from" )) { - xml_flags[inFromTag] = true; - - key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; - key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; - key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; - - /* If this is a new pair, allocate the memory */ - if ( pair == NULL ) - { - if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) - { - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - } - - pair->corr_type = manuallyNotCorrelated; - } else { - /* Otherwise, add the pair to the hash, if it's not already there */ - pair->key = key; - HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found ); - - if ( !found ) - { - HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair ); - } - - pair = NULL; - memset ( &key, 0, sizeof ( key )); - } - } else if ( !strcasecmp ((const char*) tagname, "to" )) { - xml_flags[inToTag] = true; - - key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; - key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; - key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? - strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; - - /* If this is a new pair, allocate the memory */ - if ( pair == NULL ) - { - if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) - { - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - } - - pair->corr_type = manuallyNotCorrelated; - } else { - /* Otherwise, add the pair to the hash, if it's not already there */ - pair->key = key; - HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found ); - - if ( !found ) - { - HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair ); - } - - pair = NULL; - memset ( &key, 0, sizeof ( key )); - } - } else { - AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); - } - } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { - if ( !strcasecmp ((const char*) tagname, "correlations" )) - { - if ( !xml_flags[inCorrelations] ) - { - AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelations] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { - if ( !xml_flags[inCorrelation] ) - { - AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inCorrelation] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "from" )) { - if ( !xml_flags[inFromTag] ) - { - AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inFromTag] = false; - } - } else if ( !strcasecmp ((const char*) tagname, "to" )) { - if ( !xml_flags[inToTag] ) - { - AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); - } else { - xml_flags[inToTag] = false; - } - } else { - AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); - } - } - } - - xmlFreeTextReader ( xml ); - xmlCleanupParser(); - sleep ( config->manualCorrelationsParsingInterval ); - } - - pthread_exit ((void*) 0); - return (void*) 0; -} /* ----- end of function __AI_manual_correlations_parsing_thread ----- */ - - -/** - * \brief Substitute the macros in hyperalert pre-conditions and post-conditions with their associated values - * \param alert Reference to the hyperalert to work on - */ - -PRIVATE void -__AI_macro_subst ( AI_snort_alert **alert ) -{ - /* - * Recognized macros: - * +SRC_ADDR+, +DST_ADDR+, +SRC_PORT+, +DST_PORT+, +ANY_ADDR+, +ANY_PORT+ - */ - - int i; - char src_addr[INET_ADDRSTRLEN], dst_addr[INET_ADDRSTRLEN]; - char src_port[10], dst_port[10]; - char *tmp; - - for ( i=0; i < (*alert)->hyperalert->n_preconds; i++ ) - { - tmp = (*alert)->hyperalert->preconds[i]; - (*alert)->hyperalert->preconds[i] = str_replace_all ( (*alert)->hyperalert->preconds[i], " ", "" ); - free ( tmp ); - - if ( strstr ( (*alert)->hyperalert->preconds[i], "+SRC_ADDR+" )) - { - inet_ntop ( AF_INET, &((*alert)->ip_src_addr), src_addr, INET_ADDRSTRLEN ); - tmp = (*alert)->hyperalert->preconds[i]; - (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+SRC_ADDR+", src_addr ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->preconds[i], "+DST_ADDR+" )) { - inet_ntop ( AF_INET, &((*alert)->ip_dst_addr), dst_addr, INET_ADDRSTRLEN ); - tmp = (*alert)->hyperalert->preconds[i]; - (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+DST_ADDR+", dst_addr ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->preconds[i], "+SRC_PORT+" )) { - snprintf ( src_port, sizeof ( src_port ), "%d", ntohs ((*alert)->tcp_src_port) ); - tmp = (*alert)->hyperalert->preconds[i]; - (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+SRC_PORT+", src_port ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->preconds[i], "+DST_PORT+" )) { - snprintf ( dst_port, sizeof ( dst_port ), "%d", ntohs ((*alert)->tcp_dst_port) ); - tmp = (*alert)->hyperalert->preconds[i]; - (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+DST_PORT+", dst_port ); - free ( tmp ); - } - } - - for ( i=0; i < (*alert)->hyperalert->n_postconds; i++ ) - { - tmp = (*alert)->hyperalert->postconds[i]; - (*alert)->hyperalert->postconds[i] = str_replace_all ( (*alert)->hyperalert->postconds[i], " ", "" ); - free ( tmp ); - - if ( strstr ( (*alert)->hyperalert->postconds[i], "+SRC_ADDR+" )) - { - inet_ntop ( AF_INET, &((*alert)->ip_src_addr), src_addr, INET_ADDRSTRLEN ); - tmp = (*alert)->hyperalert->postconds[i]; - (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+SRC_ADDR+", src_addr ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->postconds[i], "+DST_ADDR+" )) { - inet_ntop ( AF_INET, &((*alert)->ip_dst_addr), dst_addr, INET_ADDRSTRLEN ); - tmp = (*alert)->hyperalert->postconds[i]; - (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+DST_ADDR+", dst_addr ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->postconds[i], "+SRC_PORT+" )) { - snprintf ( src_port, sizeof ( src_port ), "%d", ntohs ((*alert)->tcp_src_port) ); - tmp = (*alert)->hyperalert->postconds[i]; - (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+SRC_PORT+", src_port ); - free ( tmp ); - } - - if ( strstr ( (*alert)->hyperalert->postconds[i], "+DST_PORT+" )) { - snprintf ( dst_port, sizeof ( dst_port ), "%d", ntohs ((*alert)->tcp_dst_port) ); - tmp = (*alert)->hyperalert->postconds[i]; - (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+DST_PORT+", dst_port ); - free ( tmp ); - } - } -} /* ----- end of function __AI_macro_subst ----- */ - -/** - * \brief Parse info about a hyperalert from a correlation XML file, if it exists - * \param key Key (gid, sid, rev) identifying the alert - * \return A hyperalert structure containing the info about the current alert, if the XML file was found - */ - -PRIVATE AI_hyperalert_info* -__AI_hyperalert_from_XML ( AI_hyperalert_key key ) -{ - char hyperalert_file[1024] = {0}; - char snort_id[1024] = {0}; - BOOL xmlFlags[HYP_TAG_NUM] = { false }; - struct stat st; - xmlTextReaderPtr xml; - const xmlChar *tagname, *tagvalue; - AI_hyperalert_info *hyp; - - if ( !( hyp = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) - { - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - } - - memset ( hyp, 0, sizeof ( AI_hyperalert_info )); - memset ( hyperalert_file, 0, sizeof ( hyperalert_file )); - - hyp->key = key; - snprintf ( hyperalert_file, sizeof ( hyperalert_file ), "%s/%d-%d-%d.xml", - config->corr_rules_dir, key.gid, key.sid, key.rev ); - - if ( stat ( hyperalert_file, &st ) < 0 ) - return NULL; - - LIBXML_TEST_VERSION - - if ( !( xml = xmlReaderForFile ( hyperalert_file, NULL, 0 ))) - return NULL; - - while ( xmlTextReaderRead ( xml )) - { - if ( !( tagname = xmlTextReaderConstName ( xml ))) - continue; - - if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) - { - if ( !strcasecmp ((const char*) tagname, "hyperalert" )) - { - if ( xmlFlags[inHyperAlert] ) - AI_fatal_err ( "Error in XML correlation rules: the hyperalert tag was opened twice", __FILE__, __LINE__ ); - else - xmlFlags[inHyperAlert] = true; - } else if ( !strcasecmp ((const char*) tagname, "snort-id" )) { - if ( xmlFlags[inSnortIdTag] ) - AI_fatal_err ( "Error in XML correlation rules: 'snort-id' tag open inside of another 'snort-id' tag", __FILE__, __LINE__ ); - else if ( !xmlFlags[inHyperAlert] ) - AI_fatal_err ( "Error in XML correlation rules: 'snort-id' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); - else - xmlFlags[inSnortIdTag] = true; - } else if ( !strcasecmp ((const char*) tagname, "pre" )) { - if ( xmlFlags[inPreTag] ) - AI_fatal_err ( "Error in XML correlation rules: 'pre' tag open inside of another 'pre' tag", __FILE__, __LINE__ ); - else if ( !xmlFlags[inHyperAlert] ) - AI_fatal_err ( "Error in XML correlation rules: 'pre' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); - else - xmlFlags[inPreTag] = true; - } else if ( !strcasecmp ((const char*) tagname, "post" )) { - if ( xmlFlags[inPostTag] ) - AI_fatal_err ( "Error in XML correlation rules: 'post' tag open inside of another 'post' tag", __FILE__, __LINE__ ); - else if ( !xmlFlags[inHyperAlert] ) - AI_fatal_err ( "Error in XML correlation rules: 'post' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); - else - xmlFlags[inPostTag] = true; - } else if ( !strcasecmp ((const char*) tagname, "desc" )) {} - else { - AI_fatal_err ( "Unrecognized tag in XML correlation rules", __FILE__, __LINE__ ); - } - } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { - if ( !strcasecmp ((const char*) tagname, "hyperalert" )) - { - if ( !xmlFlags[inHyperAlert] ) - AI_fatal_err ( "Error in XML correlation rules: hyperalert tag closed but never opend", __FILE__, __LINE__ ); - else - xmlFlags[inHyperAlert] = false; - } else if ( !strcasecmp ((const char*) tagname, "snort-id" )) { - if ( !xmlFlags[inSnortIdTag] ) - AI_fatal_err ( "Error in XML correlation rules: snort-id tag closed but never opend", __FILE__, __LINE__ ); - else - xmlFlags[inSnortIdTag] = false; - } else if ( !strcasecmp ((const char*) tagname, "pre" )) { - if ( !xmlFlags[inPreTag] ) - AI_fatal_err ( "Error in XML correlation rules: pre tag closed but never opend", __FILE__, __LINE__ ); - else - xmlFlags[inPreTag] = false; - } else if ( !strcasecmp ((const char*) tagname, "post" )) { - if ( !xmlFlags[inPostTag] ) - AI_fatal_err ( "Error in XML correlation rules: post tag closed but never opend", __FILE__, __LINE__ ); - else - xmlFlags[inPostTag] = false; - } else if ( !strcasecmp ((const char*) tagname, "desc" )) {} - else { - AI_fatal_err ( "Unrecognized tag in XML correlation rules", __FILE__, __LINE__ ); - } - } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_TEXT ) { - if ( !( tagvalue = xmlTextReaderConstValue ( xml ))) - continue; - - if ( xmlFlags[inSnortIdTag] ) - { - snprintf ( snort_id, sizeof ( snort_id ), "%d.%d.%d", - key.gid, key.sid, key.rev ); - - if ( strcmp ( snort_id, (const char*) tagvalue )) - { - _dpd.errMsg ( "AIPreproc: Found the file associated to hyperalert: '%s', " - "but the 'snort-id' field in there has a different value\n", - hyperalert_file ); - return NULL; - } - } else if ( xmlFlags[inPreTag] ) { - if ( !( hyp->preconds = (char**) realloc ( hyp->preconds, (++hyp->n_preconds)*sizeof(char*) ))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - hyp->preconds[hyp->n_preconds-1] = strdup ((const char*) tagvalue ); - } else if ( xmlFlags[inPostTag] ) { - if ( !( hyp->postconds = (char**) realloc ( hyp->postconds, (++hyp->n_postconds)*sizeof(char*) ))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - hyp->postconds[hyp->n_postconds-1] = strdup ((const char*) tagvalue ); - } - } - } - - xmlFreeTextReader ( xml ); - xmlCleanupParser(); - return hyp; -} /* ----- end of function __AI_hyperalert_from_XML ----- */ - /** * \brief Thread for correlating clustered alerts */ @@ -1240,9 +351,6 @@ AI_alert_correlation_thread ( void *arg ) AI_alert_type_pair *pair = NULL, *unpair = NULL; - AI_hyperalert_key key; - AI_hyperalert_info *hyp = NULL; - AI_snort_alert *alert_iterator = NULL, *alert_iterator2 = NULL; @@ -1263,7 +371,7 @@ AI_alert_correlation_thread ( void *arg ) pthread_mutex_init ( &mutex, NULL ); /* Start the thread for parsing manual correlations from XML */ - if ( pthread_create ( &manual_corr_thread, NULL, __AI_manual_correlations_parsing_thread, NULL ) != 0 ) + if ( pthread_create ( &manual_corr_thread, NULL, AI_manual_correlations_parsing_thread, NULL ) != 0 ) { AI_fatal_err ( "Failed to create the manual correlations parsing thread", __FILE__, __LINE__ ); } @@ -1280,7 +388,7 @@ AI_alert_correlation_thread ( void *arg ) return ( void* ) 0; } - /* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */ + /* Set the lock flag to true, and keep it this way until I've done with correlating alerts */ pthread_mutex_lock ( &mutex ); if ( alerts ) @@ -1295,46 +403,9 @@ AI_alert_correlation_thread ( void *arg ) continue; } - for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next ) + if ( config->use_knowledge_base_correlation_index != 0 ) { - /* Check if my hash table of hyperalerts already contains info about this alert */ - key.gid = alert_iterator->gid; - key.sid = alert_iterator->sid; - key.rev = alert_iterator->rev; - HASH_FIND ( hh, hyperalerts, &key, sizeof ( AI_hyperalert_key ), hyp ); - - /* If not, try to read info from the XML file, if it exists */ - if ( !hyp ) - { - /* If there is no hyperalert knowledge on XML for this alert, ignore it and get the next one */ - if ( !( hyp = __AI_hyperalert_from_XML ( key ))) - continue; - - /* If the XML file exists and it's valid, add the hypertalert to the hash table */ - HASH_ADD ( hh, hyperalerts, key, sizeof ( AI_hyperalert_key ), hyp ); - } - - /* Fill the hyper alert info for the current alert */ - if ( !( alert_iterator->hyperalert = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - alert_iterator->hyperalert->key = hyp->key; - alert_iterator->hyperalert->n_preconds = hyp->n_preconds; - alert_iterator->hyperalert->n_postconds = hyp->n_postconds; - - if ( !( alert_iterator->hyperalert->preconds = ( char** ) malloc ( alert_iterator->hyperalert->n_preconds * sizeof ( char* )))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - for ( i=0; i < alert_iterator->hyperalert->n_preconds; i++ ) - alert_iterator->hyperalert->preconds[i] = strdup ( hyp->preconds[i] ); - - if ( !( alert_iterator->hyperalert->postconds = ( char** ) malloc ( alert_iterator->hyperalert->n_postconds * sizeof ( char* )))) - AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); - - for ( i=0; i < alert_iterator->hyperalert->n_postconds; i++ ) - alert_iterator->hyperalert->postconds[i] = strdup ( hyp->postconds[i] ); - - __AI_macro_subst ( &alert_iterator ); + AI_kb_index_init ( alerts ); } __AI_correlation_table_cleanup(); @@ -1359,7 +430,7 @@ AI_alert_correlation_thread ( void *arg ) corr->correlation = 0.0; n_correlations = 0; - kb_correlation = __AI_kb_correlation_coefficient ( corr_key.a, corr_key.b ); + kb_correlation = AI_kb_correlation_coefficient ( corr_key.a, corr_key.b ); bayesian_correlation = AI_alert_bayesian_correlation ( corr_key.a, corr_key.b ); neural_correlation = AI_alert_neural_som_correlation ( corr_key.a, corr_key.b ); @@ -1370,7 +441,7 @@ AI_alert_correlation_thread ( void *arg ) n_correlations++; } - if ( kb_correlation != 0.0 ) + if ( kb_correlation != 0.0 && config->use_knowledge_base_correlation_index ) { corr->correlation += kb_correlation; n_correlations++; diff --git a/kb.c b/kb.c new file mode 100644 index 0000000..6230a3e --- /dev/null +++ b/kb.c @@ -0,0 +1,627 @@ +/* + * ===================================================================================== + * + * Filename: kb.c + * + * Description: Hyperalert knowledge-base oriented index + * + * Version: 0.1 + * Created: 20/01/2011 18:00:34 + * Revision: none + * Compiler: gcc + * + * Author: BlackLight (http://0x00.ath.cx), + * Licence: GNU GPL v.3 + * Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE! + * + * ===================================================================================== + */ + +#include "spp_ai.h" + +#include +#include +#include + +/** \defgroup correlation Module for the correlation of hyperalerts + * @{ */ + +#ifndef LIBXML_READER_ENABLED +#error "libxml2 reader not enabled\n" +#endif + +/** Enumeration for the types of hyperalert XML tags */ +enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, HYP_TAG_NUM }; + +PRIVATE AI_hyperalert_info *hyperalerts = NULL; + +/** + * \brief Get the name of the function called by a pre-condition or post-condition predicate + * \param orig_stmt Statement representing a pre-condition or post-condition + * \return The name of the function called by that statement + */ + +PRIVATE char* +__AI_get_function_name ( const char *orig_stmt ) +{ + int parenthesis_pos, function_name_len; + char function_name[4096]; + char *stmt = NULL; + + if ( !( stmt = (char*) alloca ( strlen ( orig_stmt )))) + return NULL; + strcpy ( stmt, orig_stmt ); + + memset ( function_name, 0, sizeof ( function_name )); + + if ( !( parenthesis_pos = (int) strstr ( stmt, "(" ))) + return NULL; + + parenthesis_pos -= (int) stmt; + function_name_len = ( parenthesis_pos < sizeof ( function_name )) ? parenthesis_pos : sizeof ( function_name ); + strncpy ( function_name, stmt, function_name_len ); + + return strdup(function_name); +} /* ----- end of function __AI_get_function_name ----- */ + + +/** + * \brief Get the arguments passed to a function predicate in a pre-condition or post-condition (comma-separated values) + * \param orig_stmt Statement representing a pre-condition or post-condition + * \param n_args Reference to an integer that will contain the number of arguments read + * \return An array of strings containing the arguments of the function + */ + +PRIVATE char** +__AI_get_function_arguments ( char *orig_stmt, int *n_args ) +{ + char **args = NULL; + char *tok = NULL; + char *stmt = NULL; + int par_pos = 0; + *n_args = 0; + + if ( !( stmt = (char*) alloca ( strlen ( orig_stmt )))) + return NULL; + strcpy ( stmt, orig_stmt ); + + if ( !( par_pos = (int) strstr ( stmt, "(" ))) + return NULL; + + par_pos -= (int) stmt; + stmt += par_pos + 1; + + if ( stmt [ strlen(stmt) - 1 ] == ')' ) + stmt[ strlen(stmt) - 1 ] = 0; + + tok = (char*) strtok ( stmt, "," ); + + while ( tok ) { + if ( !( args = (char**) realloc ( args, (++(*n_args)) * sizeof ( char* )))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + args [ (*n_args) - 1 ] = strdup ( tok ); + tok = (char*) strtok ( NULL, " " ); + } + + if ( !(*n_args) ) + return NULL; + + return args; +} /* ----- end of function __AI_get_function_arguments ----- */ + + +/** + * \brief Compute the correlation coefficient between two alerts, as #INTERSECTION(pre(B), post(A)) / #UNION(pre(B), post(A)), on the basis of preconditions and postconditions in the knowledge base's correlation rules + * \param a Alert a + * \param b Alert b + * \return The correlation coefficient between A and B as coefficient in [0,1] + */ + +double +AI_kb_correlation_coefficient ( const AI_snort_alert *a, const AI_snort_alert *b ) +{ + unsigned int i, j, k, l, + n_intersection = 0, + n_union = 0; + + char **args1 = NULL, + **args2 = NULL, + **matches = NULL, + *function_name1 = NULL, + *function_name2 = NULL, + new_stmt1[4096] = {0}, + new_stmt2[4096] = {0}; + + int n_args1 = 0, + n_args2 = 0, + n_matches = 0, + min_addr = 0, + max_addr = 0, + ipaddr = 0, + netmask = 0; + + if ( config->use_knowledge_base_correlation_index == 0 ) + { + return 0.0; + } + + if ( !a->hyperalert || !b->hyperalert ) + { + return 0.0; + } + + if ( a->hyperalert->n_postconds == 0 || b->hyperalert->n_preconds == 0 ) + { + return 0.0; + } + + n_union = a->hyperalert->n_postconds + b->hyperalert->n_preconds; + + for ( i=0; i < a->hyperalert->n_postconds; i++ ) + { + for ( j=0; j < b->hyperalert->n_preconds; j++ ) + { + if ( !strcasecmp ( a->hyperalert->postconds[i], b->hyperalert->preconds[j] )) + { + n_intersection += 2; + } else { + /* Check if the predicates are the same, have the same number of arguments, and + * substitute possible occurrencies of +ANY_ADDR+ and +ANY_PORT+ or IP netmasks */ + function_name1 = __AI_get_function_name ( a->hyperalert->postconds[i] ); + function_name2 = __AI_get_function_name ( b->hyperalert->preconds[j] ); + + if ( !strcasecmp ( function_name1, function_name2 )) + { + args1 = __AI_get_function_arguments ( a->hyperalert->postconds[i], &n_args1 ); + args2 = __AI_get_function_arguments ( b->hyperalert->preconds[j] , &n_args2 ); + + if ( args1 && args2 ) + { + if ( n_args1 == n_args2 ) + { + memset ( new_stmt1, 0, sizeof ( new_stmt1 )); + memset ( new_stmt2, 0, sizeof ( new_stmt2 )); + + for ( k=0; k < n_args1; k++ ) + { + /* If any occurrence of +ANY_ADDR+ or +ANY_PORT+ is found in any of the arguments, + * substitute that occurrence with the associated value */ + if ( !strcasecmp ( args1[k], "+ANY_ADDR+" ) || !strcasecmp ( args1[k], "+ANY_PORT+" )) + { + free ( args1[k] ); + args1[k] = args2[k]; + } + + if ( !strcasecmp ( args2[k], "+ANY_ADDR+" ) || !strcasecmp ( args2[k], "+ANY_PORT+" )) + { + free ( args2[k] ); + args2[k] = args1[k]; + } + + /* Substitute any occurrence of an IP netmask in any of the two arguments with + * the associated IP value */ + if ( preg_match ( "^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})/([0-9]{1,2})$", args1[k], &matches, &n_matches )) + { + if ( preg_match ( "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", args2[k], NULL, NULL )) + { + if (( netmask = strtol ( matches[1], NULL, 10 )) > 32 ) + AI_fatal_err ( "Invalid IP netmask value in configuration", __FILE__, __LINE__ ); + + if (( min_addr = inet_addr ( matches[0] )) == INADDR_NONE ) + AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); + + ipaddr = inet_addr ( args2[k] ); + + if ( ipaddr == INADDR_NONE ) + AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); + + netmask = 1 << (( 8*sizeof ( uint32_t )) - netmask ); + min_addr = ntohl ( min_addr ) & (~(netmask - 1)); + max_addr = min_addr | (netmask - 1); + ipaddr = ntohl ( ipaddr ); + + if ( ipaddr >= min_addr && ipaddr <= max_addr ) + { + free ( args1[k] ); + args1[k] = args2[k]; + } + } + + for ( l=0; l < n_matches; l++ ) + free ( matches[l] ); + free ( matches ); + } + + if ( preg_match ( "^([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})/([0-9]{1,2})$", args2[k], &matches, &n_matches )) + { + if ( preg_match ( "^[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}$", args1[k], NULL, NULL )) + { + if (( netmask = strtol ( matches[1], NULL, 10 )) > 32 ) + AI_fatal_err ( "Invalid IP netmask value in configuration", __FILE__, __LINE__ ); + + if (( min_addr = inet_addr ( matches[0] )) == INADDR_NONE ) + AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); + + ipaddr = inet_addr ( args1[k] ); + + if ( ipaddr == INADDR_NONE ) + AI_fatal_err ( "Invalid base IP address in configuration", __FILE__, __LINE__ ); + + netmask = 1 << (( 8*sizeof ( uint32_t )) - netmask ); + min_addr = ntohl ( min_addr ) & (~(netmask - 1)); + max_addr = min_addr | (netmask - 1); + ipaddr = ntohl ( ipaddr ); + + if ( ipaddr >= min_addr && ipaddr <= max_addr ) + { + free ( args2[k] ); + args2[k] = args1[k]; + } + } + + for ( l=0; l < n_matches; l++ ) + free ( matches[l] ); + free ( matches ); + } + } + + snprintf ( new_stmt1, sizeof ( new_stmt1 ), "%s(", function_name1 ); + snprintf ( new_stmt2, sizeof ( new_stmt2 ), "%s(", function_name2 ); + + for ( k=0; k < n_args1; k++ ) + { + if ( strlen ( new_stmt1 ) + strlen ( args1[k] ) + 1 < sizeof ( new_stmt1 )) + sprintf ( new_stmt1, "%s%s%s", new_stmt1, args1[k], ( k < n_args1 - 1 ) ? "," : ")" ); + + if ( strlen ( new_stmt2 ) + strlen ( args2[k] ) + 1 < sizeof ( new_stmt2 )) + sprintf ( new_stmt2, "%s%s%s", new_stmt2, args2[k], ( k < n_args2 - 1 ) ? "," : ")" ); + } + + if ( !strcmp ( new_stmt1, new_stmt2 )) + { + n_intersection += 2; + } + } + + for ( k=0; k < n_args1; k++ ) + { + if ( args1[k] ) + { + free ( args1[k] ); + args1[k] = NULL; + } + } + + if ( args1 ) + { + free ( args1 ); + args1 = NULL; + } + + for ( k=0; k < n_args2; k++ ) + { + if ( args2[k] ) + { + /* free ( args2[k] ); */ + args2[k] = NULL; + } + } + + if ( args2 ) + { + free ( args2 ); + args2 = NULL; + } + } + } + + if ( function_name1 ) + { + free ( function_name1 ); + function_name1 = NULL; + } + + if ( function_name2 ) + { + free ( function_name2 ); + function_name2 = NULL; + } + } + } + } + + return (double) ((double) n_intersection / (double) n_union ); +} /* ----- end of function __AI_kb_correlation_coefficient ----- */ + + +/** + * \brief Substitute the macros in hyperalert pre-conditions and post-conditions with their associated values + * \param alert Reference to the hyperalert to work on + */ + +PRIVATE void +__AI_macro_subst ( AI_snort_alert **alert ) +{ + /* + * Recognized macros: + * +SRC_ADDR+, +DST_ADDR+, +SRC_PORT+, +DST_PORT+, +ANY_ADDR+, +ANY_PORT+ + */ + + int i; + char src_addr[INET_ADDRSTRLEN], dst_addr[INET_ADDRSTRLEN]; + char src_port[10], dst_port[10]; + char *tmp; + + for ( i=0; i < (*alert)->hyperalert->n_preconds; i++ ) + { + tmp = (*alert)->hyperalert->preconds[i]; + (*alert)->hyperalert->preconds[i] = str_replace_all ( (*alert)->hyperalert->preconds[i], " ", "" ); + free ( tmp ); + + if ( strstr ( (*alert)->hyperalert->preconds[i], "+SRC_ADDR+" )) + { + inet_ntop ( AF_INET, &((*alert)->ip_src_addr), src_addr, INET_ADDRSTRLEN ); + tmp = (*alert)->hyperalert->preconds[i]; + (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+SRC_ADDR+", src_addr ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->preconds[i], "+DST_ADDR+" )) { + inet_ntop ( AF_INET, &((*alert)->ip_dst_addr), dst_addr, INET_ADDRSTRLEN ); + tmp = (*alert)->hyperalert->preconds[i]; + (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+DST_ADDR+", dst_addr ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->preconds[i], "+SRC_PORT+" )) { + snprintf ( src_port, sizeof ( src_port ), "%d", ntohs ((*alert)->tcp_src_port) ); + tmp = (*alert)->hyperalert->preconds[i]; + (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+SRC_PORT+", src_port ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->preconds[i], "+DST_PORT+" )) { + snprintf ( dst_port, sizeof ( dst_port ), "%d", ntohs ((*alert)->tcp_dst_port) ); + tmp = (*alert)->hyperalert->preconds[i]; + (*alert)->hyperalert->preconds[i] = str_replace ( (*alert)->hyperalert->preconds[i], "+DST_PORT+", dst_port ); + free ( tmp ); + } + } + + for ( i=0; i < (*alert)->hyperalert->n_postconds; i++ ) + { + tmp = (*alert)->hyperalert->postconds[i]; + (*alert)->hyperalert->postconds[i] = str_replace_all ( (*alert)->hyperalert->postconds[i], " ", "" ); + free ( tmp ); + + if ( strstr ( (*alert)->hyperalert->postconds[i], "+SRC_ADDR+" )) + { + inet_ntop ( AF_INET, &((*alert)->ip_src_addr), src_addr, INET_ADDRSTRLEN ); + tmp = (*alert)->hyperalert->postconds[i]; + (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+SRC_ADDR+", src_addr ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->postconds[i], "+DST_ADDR+" )) { + inet_ntop ( AF_INET, &((*alert)->ip_dst_addr), dst_addr, INET_ADDRSTRLEN ); + tmp = (*alert)->hyperalert->postconds[i]; + (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+DST_ADDR+", dst_addr ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->postconds[i], "+SRC_PORT+" )) { + snprintf ( src_port, sizeof ( src_port ), "%d", ntohs ((*alert)->tcp_src_port) ); + tmp = (*alert)->hyperalert->postconds[i]; + (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+SRC_PORT+", src_port ); + free ( tmp ); + } + + if ( strstr ( (*alert)->hyperalert->postconds[i], "+DST_PORT+" )) { + snprintf ( dst_port, sizeof ( dst_port ), "%d", ntohs ((*alert)->tcp_dst_port) ); + tmp = (*alert)->hyperalert->postconds[i]; + (*alert)->hyperalert->postconds[i] = str_replace ( (*alert)->hyperalert->postconds[i], "+DST_PORT+", dst_port ); + free ( tmp ); + } + } +} /* ----- end of function __AI_macro_subst ----- */ + +/** + * \brief Parse info about a hyperalert from a correlation XML file, if it exists + * \param key Key (gid, sid, rev) identifying the alert + * \return A hyperalert structure containing the info about the current alert, if the XML file was found + */ + +PRIVATE AI_hyperalert_info* +__AI_hyperalert_from_XML ( AI_hyperalert_key key ) +{ + char hyperalert_file[1024] = {0}; + char snort_id[1024] = {0}; + BOOL xmlFlags[HYP_TAG_NUM] = { false }; + struct stat st; + xmlTextReaderPtr xml; + const xmlChar *tagname, *tagvalue; + AI_hyperalert_info *hyp; + + if ( !( hyp = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + memset ( hyp, 0, sizeof ( AI_hyperalert_info )); + memset ( hyperalert_file, 0, sizeof ( hyperalert_file )); + + hyp->key = key; + snprintf ( hyperalert_file, sizeof ( hyperalert_file ), "%s/%d-%d-%d.xml", + config->corr_rules_dir, key.gid, key.sid, key.rev ); + + if ( stat ( hyperalert_file, &st ) < 0 ) + return NULL; + + LIBXML_TEST_VERSION + + if ( !( xml = xmlReaderForFile ( hyperalert_file, NULL, 0 ))) + return NULL; + + while ( xmlTextReaderRead ( xml )) + { + if ( !( tagname = xmlTextReaderConstName ( xml ))) + continue; + + if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) + { + if ( !strcasecmp ((const char*) tagname, "hyperalert" )) + { + if ( xmlFlags[inHyperAlert] ) + AI_fatal_err ( "Error in XML correlation rules: the hyperalert tag was opened twice", __FILE__, __LINE__ ); + else + xmlFlags[inHyperAlert] = true; + } else if ( !strcasecmp ((const char*) tagname, "snort-id" )) { + if ( xmlFlags[inSnortIdTag] ) + AI_fatal_err ( "Error in XML correlation rules: 'snort-id' tag open inside of another 'snort-id' tag", __FILE__, __LINE__ ); + else if ( !xmlFlags[inHyperAlert] ) + AI_fatal_err ( "Error in XML correlation rules: 'snort-id' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); + else + xmlFlags[inSnortIdTag] = true; + } else if ( !strcasecmp ((const char*) tagname, "pre" )) { + if ( xmlFlags[inPreTag] ) + AI_fatal_err ( "Error in XML correlation rules: 'pre' tag open inside of another 'pre' tag", __FILE__, __LINE__ ); + else if ( !xmlFlags[inHyperAlert] ) + AI_fatal_err ( "Error in XML correlation rules: 'pre' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); + else + xmlFlags[inPreTag] = true; + } else if ( !strcasecmp ((const char*) tagname, "post" )) { + if ( xmlFlags[inPostTag] ) + AI_fatal_err ( "Error in XML correlation rules: 'post' tag open inside of another 'post' tag", __FILE__, __LINE__ ); + else if ( !xmlFlags[inHyperAlert] ) + AI_fatal_err ( "Error in XML correlation rules: 'post' tag open outside of 'hyperalert' tag", __FILE__, __LINE__ ); + else + xmlFlags[inPostTag] = true; + } else if ( !strcasecmp ((const char*) tagname, "desc" )) {} + else { + AI_fatal_err ( "Unrecognized tag in XML correlation rules", __FILE__, __LINE__ ); + } + } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { + if ( !strcasecmp ((const char*) tagname, "hyperalert" )) + { + if ( !xmlFlags[inHyperAlert] ) + AI_fatal_err ( "Error in XML correlation rules: hyperalert tag closed but never opend", __FILE__, __LINE__ ); + else + xmlFlags[inHyperAlert] = false; + } else if ( !strcasecmp ((const char*) tagname, "snort-id" )) { + if ( !xmlFlags[inSnortIdTag] ) + AI_fatal_err ( "Error in XML correlation rules: snort-id tag closed but never opend", __FILE__, __LINE__ ); + else + xmlFlags[inSnortIdTag] = false; + } else if ( !strcasecmp ((const char*) tagname, "pre" )) { + if ( !xmlFlags[inPreTag] ) + AI_fatal_err ( "Error in XML correlation rules: pre tag closed but never opend", __FILE__, __LINE__ ); + else + xmlFlags[inPreTag] = false; + } else if ( !strcasecmp ((const char*) tagname, "post" )) { + if ( !xmlFlags[inPostTag] ) + AI_fatal_err ( "Error in XML correlation rules: post tag closed but never opend", __FILE__, __LINE__ ); + else + xmlFlags[inPostTag] = false; + } else if ( !strcasecmp ((const char*) tagname, "desc" )) {} + else { + AI_fatal_err ( "Unrecognized tag in XML correlation rules", __FILE__, __LINE__ ); + } + } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_TEXT ) { + if ( !( tagvalue = xmlTextReaderConstValue ( xml ))) + continue; + + if ( xmlFlags[inSnortIdTag] ) + { + snprintf ( snort_id, sizeof ( snort_id ), "%d.%d.%d", + key.gid, key.sid, key.rev ); + + if ( strcmp ( snort_id, (const char*) tagvalue )) + { + _dpd.errMsg ( "AIPreproc: Found the file associated to hyperalert: '%s', " + "but the 'snort-id' field in there has a different value\n", + hyperalert_file ); + return NULL; + } + } else if ( xmlFlags[inPreTag] ) { + if ( !( hyp->preconds = (char**) realloc ( hyp->preconds, (++hyp->n_preconds)*sizeof(char*) ))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + hyp->preconds[hyp->n_preconds-1] = strdup ((const char*) tagvalue ); + } else if ( xmlFlags[inPostTag] ) { + if ( !( hyp->postconds = (char**) realloc ( hyp->postconds, (++hyp->n_postconds)*sizeof(char*) ))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + hyp->postconds[hyp->n_postconds-1] = strdup ((const char*) tagvalue ); + } + } + } + + xmlFreeTextReader ( xml ); + xmlCleanupParser(); + return hyp; +} /* ----- end of function __AI_hyperalert_from_XML ----- */ + +/** + * \brief Initialize the hyperalert structures for the knowledge base correlation index + * \param alerts Alert list + */ + +void +AI_kb_index_init ( AI_snort_alert *alerts ) +{ + int i; + AI_hyperalert_key key; + AI_hyperalert_info *hyp = NULL; + AI_snort_alert *alert_iterator = NULL; + + if ( config->use_knowledge_base_correlation_index == 0 ) + { + return; + } + + for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next ) + { + /* Check if my hash table of hyperalerts already contains info about this alert */ + key.gid = alert_iterator->gid; + key.sid = alert_iterator->sid; + key.rev = alert_iterator->rev; + HASH_FIND ( hh, hyperalerts, &key, sizeof ( AI_hyperalert_key ), hyp ); + + /* If not, try to read info from the XML file, if it exists */ + if ( !hyp ) + { + /* If there is no hyperalert knowledge on XML for this alert, ignore it and get the next one */ + if ( !( hyp = __AI_hyperalert_from_XML ( key ))) + continue; + + /* If the XML file exists and it's valid, add the hypertalert to the hash table */ + HASH_ADD ( hh, hyperalerts, key, sizeof ( AI_hyperalert_key ), hyp ); + } + + /* Fill the hyper alert info for the current alert */ + if ( !( alert_iterator->hyperalert = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + alert_iterator->hyperalert->key = hyp->key; + alert_iterator->hyperalert->n_preconds = hyp->n_preconds; + alert_iterator->hyperalert->n_postconds = hyp->n_postconds; + + if ( !( alert_iterator->hyperalert->preconds = ( char** ) malloc ( alert_iterator->hyperalert->n_preconds * sizeof ( char* )))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + for ( i=0; i < alert_iterator->hyperalert->n_preconds; i++ ) + alert_iterator->hyperalert->preconds[i] = strdup ( hyp->preconds[i] ); + + if ( !( alert_iterator->hyperalert->postconds = ( char** ) malloc ( alert_iterator->hyperalert->n_postconds * sizeof ( char* )))) + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + + for ( i=0; i < alert_iterator->hyperalert->n_postconds; i++ ) + alert_iterator->hyperalert->postconds[i] = strdup ( hyp->postconds[i] ); + + __AI_macro_subst ( &alert_iterator ); + } +} /* ----- end of function AI_kb_index_init ----- */ + +/** @} */ + diff --git a/spp_ai.c b/spp_ai.c index ebaa122..45d7924 100644 --- a/spp_ai.c +++ b/spp_ai.c @@ -224,6 +224,7 @@ static AI_config * AI_parse(char *args) neural_train_steps = 0, output_neurons_per_side = 0, stream_expire_interval = 0, + use_knowledge_base_correlation_index = 0, use_stream_hash_table = 0, webserv_banner_len = 0, webserv_dir_len = 0; @@ -614,6 +615,27 @@ static AI_config * AI_parse(char *args) config->max_hash_pkt_number = max_hash_pkt_number; _dpd.logMsg( " Maximum number of packets stored in the hash table: %u\n", config->max_hash_pkt_number ); + /* Parsing the use_knowledge_base_correlation_index option */ + if (( arg = (char*) strcasestr( args, "use_knowledge_base_correlation_index" ) )) + { + for ( arg += strlen("use_knowledge_base_correlation_index"); + *arg && (*arg < '0' || *arg > '9'); + arg++ ); + + if ( !(*arg) ) + { + AI_fatal_err ( "use_knowledge_base_correlation_index option used but " + "no value specified", __FILE__, __LINE__ ); + } + + use_knowledge_base_correlation_index = strtoul ( arg, NULL, 10 ); + } else { + use_knowledge_base_correlation_index = DEFAULT_USE_KNOWLEDGE_BASE_CORRELATION_INDEX; + } + + config->use_knowledge_base_correlation_index = use_knowledge_base_correlation_index; + _dpd.logMsg( " Using knowledge base alert correlation index: %u\n", config->use_knowledge_base_correlation_index ); + /* Parsing the use_stream_hash_table option */ if (( arg = (char*) strcasestr( args, "use_stream_hash_table" ) )) { diff --git a/spp_ai.h b/spp_ai.h index ffdc47e..dc5b32c 100644 --- a/spp_ai.h +++ b/spp_ai.h @@ -109,6 +109,9 @@ /** Default setting for the use of the hash table for holding streams of packets * associated to a certain alert (0 = do not use, 1 or any value != 0: use) */ +#define DEFAULT_USE_KNOWLEDGE_BASE_CORRELATION_INDEX 1 + +/** Default setting for the use of the knowledge base alert correlation index */ #define DEFAULT_USE_STREAM_HASH_TABLE 1 /** Default web server port */ @@ -238,6 +241,10 @@ typedef struct /** Size of the alerts' buffer to be periodically sent to the serialization thread */ unsigned long alert_bufsize; + /** Setting for the use of the knowledge base correlation index + * (0 = do not use, 1 or any value != 0: use) */ + unsigned long use_knowledge_base_correlation_index; + /** Setting for the use of the hash table for holding streams of packets * associated to a certain alert (0 = do not use, 1 or any value != 0: use) */ unsigned long use_stream_hash_table; @@ -521,6 +528,22 @@ typedef struct { UT_hash_handle hh; } AI_geoip_cache; /*****************************************************************/ +typedef struct { + int from_gid; + int from_sid; + int from_rev; + int to_gid; + int to_sid; + int to_rev; +} AI_alert_type_pair_key; +/*****************************************************************/ +typedef struct { + AI_alert_type_pair_key key; + enum { manuallyNone, manuallyCorrelated, manuallyNotCorrelated } corr_type; + UT_hash_handle hh; +} AI_alert_type_pair; +/*****************************************************************/ + /** Enumeration for describing the table in the output database */ @@ -570,13 +593,20 @@ AI_snort_alert* AI_get_clustered_alerts ( void ); void AI_serialize_alerts ( AI_snort_alert**, unsigned int ); void AI_serializer ( AI_snort_alert* ); + void* AI_deserialize_alerts (); void* AI_alerts_pool_thread ( void* ); void* AI_neural_thread ( void* ); +void* AI_manual_correlations_parsing_thread ( void* ); +void* AI_neural_clustering_thread ( void* ); + const AI_alert_event* AI_get_alert_events_by_key ( AI_alert_event_key ); unsigned int AI_get_history_alert_number (); + double AI_alert_bayesian_correlation ( const AI_snort_alert*, const AI_snort_alert* ); double AI_alert_neural_som_correlation ( const AI_snort_alert*, const AI_snort_alert* ); +double AI_kb_correlation_coefficient ( const AI_snort_alert*, const AI_snort_alert* ); + double AI_neural_correlation_weight (); double AI_bayesian_correlation_weight (); int AI_geoinfobyaddr ( const char*, double** ); @@ -585,7 +615,7 @@ void AI_outdb_mutex_initialize (); void AI_store_alert_to_db ( AI_snort_alert* ); void AI_store_cluster_to_db ( AI_alerts_couple* ); void AI_store_correlation_to_db ( AI_alert_correlation* ); -void* AI_neural_clustering_thread ( void* ); +void AI_kb_index_init ( AI_snort_alert* ); AI_alerts_per_neuron* AI_get_alerts_per_neuron (); double(**AI_get_corr_functions ( size_t* ))(const AI_snort_alert*, const AI_snort_alert*); @@ -606,5 +636,8 @@ extern pthread_mutex_t outdb_mutex; /** Configuration of the module */ extern AI_config *config; +extern AI_alert_type_pair *manual_correlations; +extern AI_alert_type_pair *manual_uncorrelations; + #endif /* _SPP_AI_H */