From 7772c9b0893a70b83fbbb90d18fa7969aa20df2b Mon Sep 17 00:00:00 2001 From: BlackLight Date: Tue, 26 Oct 2010 21:58:34 +0200 Subject: [PATCH] Plugin support, README updated --- Makefile.am | 3 + Makefile.in | 16 ++- README | 98 +++++++++++++++-- TODO | 3 +- alert_parser.c | 2 +- cluster.c | 2 +- config.h.in | 6 + configure | 52 ++++++++- configure.ac | 3 +- corr_modules/Makefile | 3 + corr_modules/libsf_ai_corr_example.c | 37 +++++++ corr_modules/libsf_ai_corr_example.so | Bin 0 -> 4387 bytes correlation.c | 38 ++++--- modules.c | 151 ++++++++++++++++++++++++++ neural.c | 4 +- spp_ai.c | 48 +++++++- spp_ai.h | 11 +- 17 files changed, 437 insertions(+), 40 deletions(-) create mode 100644 corr_modules/Makefile create mode 100644 corr_modules/libsf_ai_corr_example.c create mode 100755 corr_modules/libsf_ai_corr_example.so create mode 100644 modules.c diff --git a/Makefile.am b/Makefile.am index 9d041e1..f41ed75 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,6 +26,7 @@ cluster.c \ correlation.c \ db.c \ fsom/fsom.c \ +modules.c \ mysql.c \ neural.c \ outdb.c \ @@ -57,6 +58,8 @@ fi mkdir -p "${SHARE_PREFIX}/htdocs" mkdir -p "${SHARE_PREFIX}/htdocs/js" mkdir -p "${SHARE_PREFIX}/schemas" + mkdir -p "${SHARE_PREFIX}/corr_modules" + cp -rf "${PWD}/corr_modules/"* "${SHARE_PREFIX}/corr_modules" install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs" diff --git a/Makefile.in b/Makefile.in index 4acff9d..e482ec3 100644 --- a/Makefile.in +++ b/Makefile.in @@ -84,11 +84,11 @@ am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_history.lo \ libsf_ai_preproc_la-cencode.lo libsf_ai_preproc_la-bayesian.lo \ libsf_ai_preproc_la-cluster.lo \ libsf_ai_preproc_la-correlation.lo libsf_ai_preproc_la-db.lo \ - libsf_ai_preproc_la-fsom.lo libsf_ai_preproc_la-mysql.lo \ - libsf_ai_preproc_la-neural.lo libsf_ai_preproc_la-outdb.lo \ - libsf_ai_preproc_la-postgresql.lo libsf_ai_preproc_la-regex.lo \ - libsf_ai_preproc_la-spp_ai.lo libsf_ai_preproc_la-stream.lo \ - libsf_ai_preproc_la-webserv.lo + libsf_ai_preproc_la-fsom.lo libsf_ai_preproc_la-modules.lo \ + libsf_ai_preproc_la-mysql.lo libsf_ai_preproc_la-neural.lo \ + libsf_ai_preproc_la-outdb.lo libsf_ai_preproc_la-postgresql.lo \ + libsf_ai_preproc_la-regex.lo libsf_ai_preproc_la-spp_ai.lo \ + libsf_ai_preproc_la-stream.lo libsf_ai_preproc_la-webserv.lo nodist_libsf_ai_preproc_la_OBJECTS = \ libsf_ai_preproc_la-sf_dynamic_preproc_lib.lo \ libsf_ai_preproc_la-sfPolicyUserData.lo @@ -268,6 +268,7 @@ cluster.c \ correlation.c \ db.c \ fsom/fsom.c \ +modules.c \ mysql.c \ neural.c \ outdb.c \ @@ -418,6 +419,9 @@ libsf_ai_preproc_la-db.lo: db.c libsf_ai_preproc_la-fsom.lo: fsom/fsom.c $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-fsom.lo `test -f 'fsom/fsom.c' || echo '$(srcdir)/'`fsom/fsom.c +libsf_ai_preproc_la-modules.lo: modules.c + $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-modules.lo `test -f 'modules.c' || echo '$(srcdir)/'`modules.c + libsf_ai_preproc_la-mysql.lo: mysql.c $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-mysql.lo `test -f 'mysql.c' || echo '$(srcdir)/'`mysql.c @@ -840,6 +844,8 @@ fi mkdir -p "${SHARE_PREFIX}/htdocs" mkdir -p "${SHARE_PREFIX}/htdocs/js" mkdir -p "${SHARE_PREFIX}/schemas" + mkdir -p "${SHARE_PREFIX}/corr_modules" + cp -rf "${PWD}/corr_modules/"* "${SHARE_PREFIX}/corr_modules" install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs" diff --git a/README b/README index 91ab2c1..e296204 100644 --- a/README +++ b/README @@ -30,7 +30,8 @@ Table of contents: 5. Correlation rules 6. Output database 7. Web interface - 8. Additional documentation + 8. Additional correlation modules + 9. Additional documentation =============================== @@ -161,18 +162,13 @@ following: preprocessor ai: \ alertfile "/your/snort/dir/log/alert" \ - alert_history_file "/your/snort/dir/log/alert_history" \ - alert_serialization_interval 3600 \ alert_bufsize 30 \ alert_clustering_interval 300 \ + alert_correlation_weight 5000 \ + alert_history_file "/your/snort/dir/log/alert_history" \ + alert_serialization_interval 3600 \ bayesian_correlation_interval 1200 \ bayesian_correlation_cache_validity 600 \ - correlation_graph_interval 300 \ - correlation_rules_dir "/your/snort/dir/etc/corr_rules" \ - correlated_alerts_dir "/your/snort/dir/log/correlated_alerts" \ - correlation_threshold_coefficient 0.5 \ - cluster_max_alert_interval 14400 \ - clusterfile "/your/snort/dir/log/clustered_alerts" \ cluster ( class="dst_port", name="privileged_ports", range="1-1023" ) \ cluster ( class="dst_port", name="unprivileged_ports", range="1024-65535" ) \ cluster ( class="src_addr", name="local_net", range="192.168.1.0/24" ) \ @@ -181,10 +177,21 @@ preprocessor ai: \ cluster ( class="dst_addr", name="local_net", range="192.168.1.0/24" ) \ cluster ( class="dst_addr", name="dmz_net", range="155.185.0.0/16" ) \ cluster ( class="dst_addr", name="vpn_net", range="10.8.0.0/24" ) \ + cluster_max_alert_interval 14400 \ + clusterfile "/your/snort/dir/log/clustered_alerts" \ + corr_modules_dir "/your/snort/dir/share/snort_ai_preproc/corr_modules" \ + correlation_graph_interval 300 \ + correlation_rules_dir "/your/snort/dir/etc/corr_rules" \ + correlated_alerts_dir "/your/snort/dir/log/correlated_alerts" \ + correlation_threshold_coefficient 0.5 \ database ( type="dbtype", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \ database_parsing_interval 30 \ hashtable_cleanup_interval 300 \ + manual_correlations_parsing_interval 120 \ + neural_network_training_interval 43200 \ + neural_train_steps 10 \ output_database ( type="dbtype", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \ + output_neurons_per_side 20 \ tcp_stream_expire_interval 300 \ webserv_banner "Snort AIPreprocessor module" \ webserv_dir "/prefix/share/htdocs" \ @@ -197,6 +204,15 @@ The options are the following: and not to a database (default if not specified: /var/log/snort/alert) +- alert_correlation_weight: When this number of alert is stored in the "memory" +of the software (i.e. in the alert history file or in the output database), the +weight for the heuristical correlation indexes (bayesian network and neural + network) will be more or less equal to 0.95, on a scale from 0 to 1. +This parameter expresses how much the heuristical indexes should be weighted and +it can be considered like a kind of "learning rate" for the alert correlation +algorithm (default value if not specified: 5000) + + - alert_history_file: The file keeping track of the history, in binary format, of all the alerts received by the IDS, so that the module can build some statistical correlation inferences over the past @@ -234,6 +250,14 @@ in the bayesian correlation hash table (i.e. a pair of alerts with the before being updated (default: 600 seconds) +- corr_modules_dir: This software supports a kind of plugins, or "modules over +the module", that allow the user to specify some extra correlation rules and +indexes. These modules are .so files placed in this directory (default if not + specified: PREFIX/share/snort_ai_preproc/corr_modules), dynamically loaded by +the module. For more information on how to write your own module, see the +dedicated section in this file. + + - correlation_graph_interval: The interval that should occur from the building of the correlation graph between the clustered alerts and the next one (default if not specified: 300 seconds) @@ -301,6 +325,20 @@ the hashtable of TCP streams and the next one (default if not specified: 300 seconds) +- manual_correlations_parsing_interval: Interval in seconds between an execution +of the thread for parsing the alert correlations manually set and the next one +(default value if not specified: 120 seconds) + + +- neural_network_training_interval: Interval in seconds between an execution of +the thread for training the neural network using the set of recent alerts and +the next one (default if not specified: 43200 seconds) + + +- neural_train_steps: Number of steps to take in each training cycle for the +neural network (default: 10) + + - output_database: Specify this option if you want to save the outputs from the module (correlated alerts, clustered alerts, alerts information and their associated packets streams, and so on) to a relational database as @@ -312,6 +350,13 @@ module, just give the right file to your database, e.g. for MySQL $ mysql -uusername -ppassword dbname < schemas/mysql.sql +- output_neurons_per_side: Number of output neurons per side on the output layer +of the neural network (that is a rectangular matrix). A higher number allows a +higher granularity over similar alerts, but a linear increment of this value +produces a squared increment of the computational complexity for the training +and evaluation algorithms (default value if not specified: 20) + + - tcp_stream_expire_interval: The interval that should occur for marking a TCP stream as "expired", if no more packets are received inside of that and it's not "marked" as suspicious (default if not specified: 300 seconds) @@ -467,8 +512,41 @@ that, all the alerts of those types will be marted as correlated, or uncorrelated. +================================= +8. Additional correlation modules +================================= + +It is possible to add extra parameters and indexes for evaluating the +correlation between two alerts in an extremely simple way. The directory +specified in the configuration option "corr_modules_dir" contains the extra +modules (as binary shared libraries -> .so). Each of these modules should +contain a function whose prototype is + + +double AI_corr_index ( AI_snort_alert*, AI_snort_alert* ) + + +taking two alerts as parameters and returning a correlation value between them, +and one whose prototype is + + +double AI_corr_index_weight () + + +returning a coefficient in [0,1] expressing the weight of that index. An +example module is contained in the corr_modules directory in the source +directory, or in PREFIX/share/snort_ai_preproc/corr_modules after installation. + +When you write your own module, just add in the Makefile in the corr_modules +directory a line like the one already present there for compiling, then type +`make'. You may need to link your module source file(s) against +libsf_ai_preproc.la if you want to use some of the functions from the module, + for example, for reading the alerts stored in the history file, in the + database, the current correlations, and so on. + + =========================== -8. Additional documentation +9. Additional documentation =========================== The additional documentation over the code, functions and data structures can diff --git a/TODO b/TODO index 952042b..4935c88 100644 --- a/TODO +++ b/TODO @@ -2,10 +2,8 @@ AVERAGE/HIGH PRIORITY: ====================== -- Supporting extra modules for alert correlation - Code profiling - Comment all the code!!! -- Neural network for computing k - Testing more scenarios, making more hyperalert models ============= @@ -36,4 +34,5 @@ DONE: + Saving packet flows as .pcap + Manual alert correlation from the web interface + Neural network for alert correlation ++ Supporting extra modules for alert correlation diff --git a/alert_parser.c b/alert_parser.c index 805d87d..f5cc6db 100644 --- a/alert_parser.c +++ b/alert_parser.c @@ -497,7 +497,7 @@ __AI_copy_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_alerts () { - AI_snort_alert *alerts_copy; + AI_snort_alert *alerts_copy = NULL; pthread_mutex_lock ( &alert_mutex ); alerts_copy = __AI_copy_alerts ( alerts ); diff --git a/cluster.c b/cluster.c index e307444..894cfcc 100644 --- a/cluster.c +++ b/cluster.c @@ -751,7 +751,7 @@ __AI_copy_clustered_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_clustered_alerts () { - AI_snort_alert *alerts_copy; + AI_snort_alert *alerts_copy = NULL; pthread_mutex_lock ( &mutex ); alerts_copy = __AI_copy_clustered_alerts ( alert_log ); diff --git a/config.h.in b/config.h.in index 4167cee..5b3be33 100644 --- a/config.h.in +++ b/config.h.in @@ -42,6 +42,9 @@ /* Define to 1 if you have the `connect' function. */ #undef HAVE_CONNECT +/* Define to 1 if you have the header file. */ +#undef HAVE_DIRENT_H + /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H @@ -60,6 +63,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H +/* Define to 1 if you have the `dl' library (-ldl). */ +#undef HAVE_LIBDL + /* Define to 1 if you have the `gvc' library (-lgvc). */ #undef HAVE_LIBGVC diff --git a/configure b/configure index fb8e775..6dbb826 100755 --- a/configure +++ b/configure @@ -11719,6 +11719,56 @@ as_fn_error $? "libm not found on the system See \`config.log' for more details" "$LINENO" 5 ; } fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 +$as_echo_n "checking for dlopen in -ldl... " >&6; } +if test "${ac_cv_lib_dl_dlopen+set}" = set; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlopen (); +int +main () +{ +return dlopen (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dl_dlopen=yes +else + ac_cv_lib_dl_dlopen=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 +$as_echo "$ac_cv_lib_dl_dlopen" >&6; } +if test "x$ac_cv_lib_dl_dlopen" = x""yes; then : + cat >>confdefs.h <<_ACEOF +#define HAVE_LIBDL 1 +_ACEOF + + LIBS="-ldl $LIBS" + +else + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "libdl not found on the system +See \`config.log' for more details" "$LINENO" 5 ; } +fi + if test "x$prefix" == x/usr; then : CORR_RULES_PREFIX="/etc/snort/corr_rules" @@ -11956,7 +12006,7 @@ _ACEOF fi -for ac_header in inttypes.h limits.h stddef.h stdlib.h string.h unistd.h wchar.h math.h +for ac_header in dirent.h dlfcn.h inttypes.h limits.h math.h stddef.h stdlib.h string.h unistd.h wchar.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" diff --git a/configure.ac b/configure.ac index f30d550..7990642 100644 --- a/configure.ac +++ b/configure.ac @@ -136,6 +136,7 @@ AS_IF([test "x$with_graphviz" != xno], AC_CHECK_LIB([xml2], [xmlReaderForFile],, AC_MSG_FAILURE(libxml2 not found on the system)) AC_CHECK_LIB([pthread], [pthread_create],, AC_MSG_FAILURE(libpthread not found on the system)) AC_CHECK_LIB([m], [sqrt],, AC_MSG_FAILURE(libm not found on the system)) +AC_CHECK_LIB([dl], [dlopen],, AC_MSG_FAILURE(libdl not found on the system)) AS_IF([test "x$prefix" == x/usr], [AC_SUBST([CORR_RULES_PREFIX], ["/etc/snort/corr_rules"])], @@ -161,7 +162,7 @@ AS_IF([test "x$with_graphviz" != xno], [AC_DEFINE([HAVE_BOOLEAN], [1], [Check if the boolean type is defined])]) AC_FUNC_ALLOCA -AC_CHECK_HEADERS([inttypes.h limits.h stddef.h stdlib.h string.h unistd.h wchar.h math.h],,AC_MSG_ERROR(At least one of the required headers was not found)) +AC_CHECK_HEADERS([dirent.h dlfcn.h inttypes.h limits.h math.h stddef.h stdlib.h string.h unistd.h wchar.h],,AC_MSG_ERROR(At least one of the required headers was not found)) # Check for int types AC_CHECK_TYPES([u_int8_t,u_int16_t,u_int32_t,u_int64_t,uint8_t,uint16_t,uint32_t,uint64_t]) diff --git a/corr_modules/Makefile b/corr_modules/Makefile new file mode 100644 index 0000000..d39a2aa --- /dev/null +++ b/corr_modules/Makefile @@ -0,0 +1,3 @@ +all: + gcc -I. -I.. -I../uthash -I../base64 -I../fsom -I../include -D_GNU_SOURCE -Wall -pedantic -pedantic-errors -std=c99 -fPIC -shared -rdynamic -o libsf_ai_corr_example.so libsf_ai_corr_example.c + diff --git a/corr_modules/libsf_ai_corr_example.c b/corr_modules/libsf_ai_corr_example.c new file mode 100644 index 0000000..c13746a --- /dev/null +++ b/corr_modules/libsf_ai_corr_example.c @@ -0,0 +1,37 @@ +/* + * ===================================================================================== + * + * Filename: libsf_ai_corr_example.c + * + * Description: Sample correlation module for two alerts + * + * Version: 0.1 + * Created: 26/10/2010 14:23:24 + * Revision: none + * Compiler: gcc + * + * Author: BlackLight (http://0x00.ath.cx), + * Licence: GNU GPL v.3 + * Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE! + * + * ===================================================================================== + */ + +#include "spp_ai.h" + +/** Function that, given two alerts, returns a correlation index in [0,1] */ + +double +AI_corr_index ( const AI_snort_alert *a, const AI_snort_alert *b ) +{ + return 0.5; +} + +/** Function that returns the weight of this index */ + +double +AI_corr_index_weight () +{ + return 0.0; +} + diff --git a/corr_modules/libsf_ai_corr_example.so b/corr_modules/libsf_ai_corr_example.so new file mode 100755 index 0000000000000000000000000000000000000000..751140e09385949e4e65e7f0d2a700fa1d57868f GIT binary patch literal 4387 zcmb7HYiv|S6rSA{d{wNqDk|!#jj2&@5K9FUwWXAb6sS-Pig>-d_qKa=U+&(;1)JEq zH7Qvd8zIK%9}%Pe(V(dDnLrke`bXj`{4voWW{pH4QB#fK`h7EZ>fNd_p3T{FzBy;+ z%;U`5nTLD(S41Kap{YSM2%^0WLUcfmo}w%rA|~dDcCkQQAnn?p(wn0gLWpJ|?Xv7# zl!ND^!5DaMV2?shza&_lskPAK17c`uqnsy)}7a-i_~1Z~Ni1pXR|_M~tZ{2IXoEAc+@* z@a51S2Jbpg6*h|#ehTW25H8HZtAQUu=jhTRnz&42AQ*HDMtTcyDrBeMVGah=BF+@T zG>6mKj9Kukyl0xibTh6e%pnMv5+#eOm2$T^!n|#Zxy~7O3!anjNm+%0QxM%N%|tez zM=jea)g$vk#~n_3BIU*tMj>l-3e&MI&qA~DLO}&YQN|X1{VSLCn2U_-jO$g$F#D0Z zA9F7v;;?djq{5{xmLe*~6%;uPjBD-%vJ}XXK#m4-JdhKC9DtmQMH-~UqHm&wR)YYew(tSZhCqO$X6-}fK=)SFYg1f{jN_}7Ly8#jGa_4jd-{XOgx z$=}n2|0LD#zeY8G4@Z)u`TIkFX#bA&wU+?Ls?W9vQ9jl@`p!Z?QQTZ9|G_}}y?K=~ z^M89|TkTzxRLUJdll$7BPwtH2|Eez!oQ#%#j4VA~h)ur1+U5O`?b6NPRi=lkv-EiW zyAR52K2ko|yy>3JRgSM_pkS-&q{jOm+8wLD3VWqIA36eg z7fkM>I{Dg_s4U7qHu{Gub73g{tos>ShZ3sR_Mi0qJ2B_T*~9%_BL@0)Gox12~GnRtEr z+nZU2Ui)E59qAzRmaETwi5(~RAG2c`{L4u{y97==gW8dQJY!10Jfo(-Jj<9Vs4DJ8 zF2Y4(<^jeo&IfnFNc}jBSUWKQMt;iufT_tILOHsYWnYd%z6tz0Wbg?<>Hj01?V6rI z?B~JUf72LL6a#4iUk1J&+(+)|xiQwh+=;tZCe~r#ckK;VDMbjQur=*jaY!$(WKwIH ztmhcRnW7Odx+(iw*A@~btwK^5_SOt)D(U5wWee8X&1UL=2^-dr{z}QEJYmRF&hVTP zba~bp`K&yg3@2$0<*l?MjD(j(+8DMH_a)E}8nM!D0s*q9h(ANNrfT9;L#1pw?PNSX zJ^v{$f(O@u>-u@X+)J96C!|GSOr`W=-jKV%+D=L47XB4$%l4R8L+XugT2o@HlvQrbB+*2W0k}ejFbq{kDMFSMD$FL-IoCnsmDlpg{YL!iPNDkG>-z zzrFAwx9iD(jeQ_J0@m&EyGss#^L5;bzz@Te`-pp~1YHxyK$-%xJ^JM^X=L37ie~hu zpwkcCl6N)U+jJW!n(^BWUHgq-UyuBY--{u?arlk@OI&`zILC}D?}7095ITe;_Lcp3 z1FYln8$5tYJR>#HkNgf;`|+tX`Lzi@lBk$LX2Eln@N@301t^dV7l%`|>9Fgu{1pT`Z zSobf2tZ4_<{fpqMy%?Bh2q}u4unLm#xI_`ey9;s(m{UZYgj^WR7yZL|!T28RrC|Jw zKL*URf)o*FiN_$r@t+2MCe+?5bPR~?eFU9%>-mrooa|ph>iLR9Px!}x^?XI5=ZW8d zM|J$@+47G4q;5ajBrZmWheNmnSl=ty-c7*acgreZO))5U&B7UAoj-inc))tS_>LL{ z4!^4&1rEQPo|@&q8(7ojP~HRPH-p4?l}O-@=@o~Dj07GuJ%el4nf)u*5Ap?NF0Vtk z^@L1z)*Mb{<5tR)*9y}rmISU0xs-$ZrLlPF;!Z(>>DnctB|_dsOuLv)Z`E;n*DRM| zR;=q@-D_&ZFk1gBioC9B zSl>Hn4t6i=@8yW>Le@-L89Rknpckey.a->tcp_src_port )); snprintf ( dst_port1, sizeof ( dst_port1 ), "%d", ntohs ( corr->key.a->tcp_dst_port )); - timestamp1 = ctime ( &(corr->key.a->timestamp ) ); - timestamp1 [ strlen ( timestamp1 ) - 1 ] = 0; - inet_ntop ( AF_INET, &(corr->key.b->ip_src_addr), src_addr2, INET_ADDRSTRLEN ); inet_ntop ( AF_INET, &(corr->key.b->ip_dst_addr), dst_addr2, INET_ADDRSTRLEN ); snprintf ( src_port2, sizeof ( src_port2 ), "%d", ntohs ( corr->key.b->tcp_src_port )); snprintf ( dst_port2, sizeof ( dst_port2 ), "%d", ntohs ( corr->key.b->tcp_dst_port )); - timestamp2 = ctime ( &(corr->key.b->timestamp ) ); - timestamp2 [ strlen ( timestamp2 ) - 1 ] = 0; - fprintf ( fp, "\t\"[%d.%d.%d] %s\\n" "%s:%s -> %s:%s\\n" - "%s\\n" "(%d alerts grouped)\" -> " "\"[%d.%d.%d] %s\\n" "%s:%s -> %s:%s\\n" - "%s\\n" "(%d alerts grouped)\";\n", corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc, src_addr1, src_port1, dst_addr1, dst_port1, - timestamp1, corr->key.a->grouped_alerts_count, corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc, src_addr2, src_port2, dst_addr2, dst_port2, - timestamp2, corr->key.b->grouped_alerts_count ); } /* ----- end of function __AI_correlated_alerts_to_dot ----- */ @@ -1230,7 +1218,10 @@ AI_alert_correlation_thread ( void *arg ) bayesian_correlation = 0.0, neural_correlation = 0.0; - size_t n_correlations = 0; + size_t n_correlations = 0, + n_corr_functions = 0, + n_corr_weights = 0; + FILE *fp = NULL; AI_alert_correlation_key corr_key; @@ -1255,6 +1246,12 @@ AI_alert_correlation_thread ( void *arg ) graph_t *g = NULL; #endif + double (**corr_functions)( const AI_snort_alert*, const AI_snort_alert* ) = NULL; + double (**corr_weights)() = NULL; + + corr_functions = AI_get_corr_functions( &n_corr_functions ); + corr_weights = AI_get_corr_weights ( &n_corr_weights ); + pthread_mutex_init ( &mutex, NULL ); /* Start the thread for parsing manual correlations from XML */ @@ -1377,6 +1374,19 @@ AI_alert_correlation_thread ( void *arg ) n_correlations++; } + /* Get the correlation indexes from extra correlation modules */ + if (( corr_functions )) + { + for ( i=0; i < n_corr_functions; i++ ) + { + if ( corr_weights[i]() != 0.0 ) + { + corr->correlation += corr_weights[i]() * corr_functions[i] ( corr_key.a, corr_key.b ); + n_correlations++; + } + } + } + if ( n_correlations != 0 ) { corr->correlation /= (double) n_correlations; diff --git a/modules.c b/modules.c new file mode 100644 index 0000000..0b8b97c --- /dev/null +++ b/modules.c @@ -0,0 +1,151 @@ +/* + * ===================================================================================== + * + * Filename: modules.c + * + * Description: Support for extra correlation modules + * + * Version: 0.1 + * Created: 26/10/2010 01:11:25 + * Revision: none + * Compiler: gcc + * + * Author: BlackLight (http://0x00.ath.cx), + * Licence: GNU GPL v.3 + * Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE! + * + * ===================================================================================== + */ + +#include "spp_ai.h" + +#include +#include +#include +#include + +/** \defgroup modules Software component for loading extra user-provided modules for correlating alerts + * @{ */ + +PRIVATE double (**corr_functions)(const AI_snort_alert*, const AI_snort_alert*) = NULL; +PRIVATE size_t n_corr_functions = 0; + +PRIVATE double (**weight_functions)() = NULL; +PRIVATE size_t n_weight_functions = 0; + +/** + * \brief Get the correlation functions from the extra correlation modules as array of function pointers + * \param n_functions Number of function pointers in the array + * \return The array of correlation functions + */ + +double +(**AI_get_corr_functions ( size_t *n_functions )) (const AI_snort_alert*, const AI_snort_alert*) +{ + *n_functions = n_corr_functions; + return corr_functions; +} /* ----- end of function AI_get_corr_functions ----- */ + +/** + * \brief Get the weights of the correlation extra modules as array of function pointers + * \param n_functions Number of function pointers in the array + * \return The array of correlation weights functions + */ + +double +(**AI_get_corr_weights ( size_t *n_functions )) () +{ + *n_functions = n_weight_functions; + return weight_functions; +} /* ----- end of function AI_get_corr_weights ----- */ + +/** + * \brief Initialize the extra modules provided by the user + */ + +void +AI_init_corr_modules () +{ + void **dl_handles = NULL; + DIR *dir = NULL; + char *err = NULL; + char *fname = NULL; + size_t n_dl_handles = 0; + struct dirent *dir_info = NULL; + + if ( !( dir = opendir ( config->corr_modules_dir ))) + { + return; + } + + while (( dir_info = readdir ( dir ))) + { + if ( preg_match ( "(\\.(l|s)o)|(\\.l?a)", dir_info->d_name, NULL, NULL )) + { + if ( !( dl_handles = (void**) realloc ( dl_handles, (++n_dl_handles) * sizeof ( void* )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + if ( !( fname = (char*) malloc ( strlen ( config->corr_modules_dir ) + strlen ( dir_info->d_name ) + 4 ))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + sprintf ( fname, "%s/%s", config->corr_modules_dir, dir_info->d_name ); + + if ( !( dl_handles[n_dl_handles-1] = dlopen ( fname, RTLD_LAZY ))) + { + if (( err = dlerror() )) + { + _dpd.errMsg ( "dlopen: %s\n", err ); + } + + AI_fatal_err ( "dlopen error", __FILE__, __LINE__ ); + } + + free ( fname ); + fname = NULL; + + if ( !( corr_functions = (double(**)(const AI_snort_alert*, const AI_snort_alert*)) + realloc ( corr_functions, (++n_corr_functions) * sizeof ( double(*)(const AI_snort_alert*, const AI_snort_alert*) )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + *(void**) (&(corr_functions[ n_corr_functions - 1 ])) = dlsym ( dl_handles[n_dl_handles-1], "AI_corr_index" ); + + if ( !corr_functions[ n_corr_functions - 1 ] ) + { + if (( err = dlerror() )) + { + _dpd.errMsg ( "dlsym: %s\n", err ); + } + + AI_fatal_err ( "dlsym error", __FILE__, __LINE__ ); + } + + if ( !( weight_functions = (double(**)()) realloc ( weight_functions, (++n_weight_functions) * sizeof ( double(*)() )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + *(void**) (&(weight_functions[ n_weight_functions - 1 ])) = dlsym ( dl_handles[n_dl_handles-1], "AI_corr_index_weight" ); + + if ( !weight_functions[ n_weight_functions - 1 ] ) + { + if (( err = dlerror() )) + { + _dpd.errMsg ( "dlsym: %s\n", err ); + } + + AI_fatal_err ( "dlsym error", __FILE__, __LINE__ ); + } + } + } + + closedir ( dir ); +} /* ----- end of function AI_init_corr_modules ----- */ + +/** @} */ + diff --git a/neural.c b/neural.c index 1ef9c16..fff97d0 100644 --- a/neural.c +++ b/neural.c @@ -251,7 +251,7 @@ __AI_som_train () if ( !( res = (DB_result) DB_out_query ( query ))) { - AI_fatal_err ( "AIPreproc: Query error", __FILE__, __LINE__ ); + return; } num_rows = DB_num_rows ( res ); @@ -277,7 +277,7 @@ __AI_som_train () { row = (DB_row) DB_fetch_row ( res ); - tuples[i].gid = row[0] ? strtoul ( row[0], NULL, 10 ) : 0; + tuples[i].sid = row[1] ? strtoul ( row[1], NULL, 10 ) : 0; tuples[i].rev = row[2] ? strtoul ( row[2], NULL, 10 ) : 0; tuples[i].timestamp = row[3] ? (time_t) strtol ( row[3], NULL, 10 ) : (time_t) 0; diff --git a/spp_ai.c b/spp_ai.c index 1834995..fb0cbd3 100644 --- a/spp_ai.c +++ b/spp_ai.c @@ -110,6 +110,9 @@ static void AI_init(char *args) sfPolicyUserPolicySet(ex_config, policy_id); sfPolicyUserDataSetCurrent(ex_config, config); + /* Initialize the extra correlation modules */ + AI_init_corr_modules(); + /* If the hash_cleanup_interval or stream_expire_interval options are set to zero, * no cleanup will be made on the streams */ if ( config->hashCleanupInterval != 0 && config->streamExpireInterval != 0 ) @@ -173,8 +176,9 @@ static AI_config * AI_parse(char *args) char alertfile[1024] = { 0 }, alert_history_file[1024] = { 0 }, clusterfile[1024] = { 0 }, - corr_rules_dir[1024] = { 0 }, corr_alerts_dir[1024] = { 0 }, + corr_modules_dir[1024] = { 0 }, + corr_rules_dir[1024] = { 0 }, webserv_dir[1024] = { 0 }, webserv_banner[1024] = { 0 }; @@ -209,6 +213,7 @@ static AI_config * AI_parse(char *args) clusterfile_len = 0, cluster_max_alert_interval = 0, corr_alerts_dir_len = 0, + corr_modules_dir_len = 0, corr_rules_dir_len = 0, correlation_graph_interval = 0, database_parsing_interval = 0, @@ -224,6 +229,7 @@ static AI_config * AI_parse(char *args) has_stream_expire_interval = false, has_correlation_interval = false, has_corr_alerts_dir = false, + has_corr_modules_dir = false, has_database_interval = false, has_webserv_dir = false, has_webserv_banner = false, @@ -791,6 +797,46 @@ static AI_config * AI_parse(char *args) _dpd.logMsg(" webserv_dir: %s\n", config->webserv_dir); + /* Parsing the corr_modules_dir option */ + if (( arg = (char*) strcasestr( args, "corr_modules_dir" ) )) + { + for ( arg += strlen("corr_modules_dir"); + *arg && *arg != '"'; + arg++ ); + + if ( !(*(arg++)) ) + { + AI_fatal_err ( "corr_modules_dir option used but no filename specified", __FILE__, __LINE__ ); + } + + for ( corr_modules_dir[ (++corr_modules_dir_len)-1 ] = *arg; + *arg && *arg != '"' && corr_modules_dir_len < sizeof ( corr_modules_dir ); + arg++, corr_modules_dir[ (++corr_modules_dir_len)-1 ] = *arg ); + + if ( corr_modules_dir[0] == 0 || corr_modules_dir_len <= 1 ) { + has_corr_modules_dir = false; + } else { + if ( corr_modules_dir_len >= sizeof ( corr_modules_dir )) { + AI_fatal_err ( "corr_modules_dir path too long ( >= 1024 )", __FILE__, __LINE__ ); + } else if ( strlen( corr_modules_dir ) == 0 ) { + has_corr_modules_dir = false; + } else { + has_corr_modules_dir = true; + corr_modules_dir[ corr_modules_dir_len-1 ] = 0; + strncpy ( config->corr_modules_dir, corr_modules_dir, corr_modules_dir_len ); + } + } + } + + if ( ! has_corr_modules_dir ) + { + #ifndef HAVE_CONFIG_H + AI_fatal_err ( "Unable to read PREFIX from config.h", __FILE__, __LINE__ ); + #endif + + snprintf ( config->corr_modules_dir, sizeof ( config->corr_modules_dir ), "%s/share/snort_ai_preprocessor/corr_modules", PREFIX ); + } + /* Neural network output file */ if ( config->neuralNetworkTrainingInterval != 0 ) { diff --git a/spp_ai.h b/spp_ai.h index 9775192..56a3500 100644 --- a/spp_ai.h +++ b/spp_ai.h @@ -94,7 +94,7 @@ /** Default number of alerts needed in the history file or database for letting a certain * heuristic correlation index weight be =~ 0.95 (the weight monotonically increases * with the number of alerts according to a hyperbolic tangent function) */ -#define DEFAULT_ALERT_CORRELATION_WEIGHT 400 +#define DEFAULT_ALERT_CORRELATION_WEIGHT 5000 /** Default web server port */ #define DEFAULT_WEBSERV_PORT 7654 @@ -105,7 +105,7 @@ /** Cutoff y value in the exponential decay for considering two alerts not correlated */ #define CUTOFF_Y_VALUE 0.01 -/** Approximated solution of the equation tanh(x) = 0.95 */ +/** Approximated solution of the equation tanh(x) = 0.95 (used as parameter in the correlation indexes weight function) */ #define HYPERBOLIC_TANGENT_SOLUTION 1.83178 /****************************/ @@ -233,6 +233,9 @@ typedef struct * and in the footer of error pages */ char webserv_banner[1024]; + /** Directory containing extra correlation modules */ + char corr_modules_dir[1024]; + /** Alert file */ char alertfile[1024]; @@ -486,6 +489,7 @@ void AI_pkt_enqueue ( SFSnortPacket* ); void AI_set_stream_observed ( struct pkt_key key ); void AI_hierarchies_build ( hierarchy_node**, int ); void AI_free_alerts ( AI_snort_alert *node ); +void AI_init_corr_modules (); struct pkt_info* AI_get_stream_by_key ( struct pkt_key ); AI_snort_alert* AI_get_alerts ( void ); @@ -508,6 +512,9 @@ void* AI_store_alert_to_db_thread ( void* ); void* AI_store_cluster_to_db_thread ( void* ); void* AI_store_correlation_to_db_thread ( void* ); +double(**AI_get_corr_functions ( size_t* ))(const AI_snort_alert*, const AI_snort_alert*); +double(**AI_get_corr_weights ( size_t* ))(); + /** Function pointer to the function used for getting the alert list (from log file, db, ...) */ extern AI_snort_alert* (*get_alerts)(void);