Plugin support, README updated

Makefile.am

@@ -26,6 +26,7 @@ cluster.c \
 correlation.c \
 db.c \
 fsom/fsom.c \
+modules.c \
 mysql.c \
 neural.c \
 outdb.c \
@@ -57,6 +58,8 @@ fi
 	mkdir -p "${SHARE_PREFIX}/htdocs"
 	mkdir -p "${SHARE_PREFIX}/htdocs/js"
 	mkdir -p "${SHARE_PREFIX}/schemas"
+	mkdir -p "${SHARE_PREFIX}/corr_modules"
+	cp -rf "${PWD}/corr_modules/"* "${SHARE_PREFIX}/corr_modules"
 	install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs"
 	install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs"
 	install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs"

Makefile.in

@@ -84,11 +84,11 @@ am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_history.lo \
 	libsf_ai_preproc_la-cencode.lo libsf_ai_preproc_la-bayesian.lo \
 	libsf_ai_preproc_la-cluster.lo \
 	libsf_ai_preproc_la-correlation.lo libsf_ai_preproc_la-db.lo \
-	libsf_ai_preproc_la-fsom.lo libsf_ai_preproc_la-mysql.lo \
+	libsf_ai_preproc_la-fsom.lo libsf_ai_preproc_la-modules.lo \
-	libsf_ai_preproc_la-neural.lo libsf_ai_preproc_la-outdb.lo \
+	libsf_ai_preproc_la-mysql.lo libsf_ai_preproc_la-neural.lo \
-	libsf_ai_preproc_la-postgresql.lo libsf_ai_preproc_la-regex.lo \
+	libsf_ai_preproc_la-outdb.lo libsf_ai_preproc_la-postgresql.lo \
-	libsf_ai_preproc_la-spp_ai.lo libsf_ai_preproc_la-stream.lo \
+	libsf_ai_preproc_la-regex.lo libsf_ai_preproc_la-spp_ai.lo \
-	libsf_ai_preproc_la-webserv.lo
+	libsf_ai_preproc_la-stream.lo libsf_ai_preproc_la-webserv.lo
 nodist_libsf_ai_preproc_la_OBJECTS =  \
 	libsf_ai_preproc_la-sf_dynamic_preproc_lib.lo \
 	libsf_ai_preproc_la-sfPolicyUserData.lo
@@ -268,6 +268,7 @@ cluster.c \
 correlation.c \
 db.c \
 fsom/fsom.c \
+modules.c \
 mysql.c \
 neural.c \
 outdb.c \
@@ -418,6 +419,9 @@ libsf_ai_preproc_la-db.lo: db.c
 libsf_ai_preproc_la-fsom.lo: fsom/fsom.c
 	$(LIBTOOL)  --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-fsom.lo `test -f 'fsom/fsom.c' || echo '$(srcdir)/'`fsom/fsom.c
 
+libsf_ai_preproc_la-modules.lo: modules.c
+	$(LIBTOOL)  --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-modules.lo `test -f 'modules.c' || echo '$(srcdir)/'`modules.c
+
 libsf_ai_preproc_la-mysql.lo: mysql.c
 	$(LIBTOOL)  --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-mysql.lo `test -f 'mysql.c' || echo '$(srcdir)/'`mysql.c
 
@@ -840,6 +844,8 @@ fi
 	mkdir -p "${SHARE_PREFIX}/htdocs"
 	mkdir -p "${SHARE_PREFIX}/htdocs/js"
 	mkdir -p "${SHARE_PREFIX}/schemas"
+	mkdir -p "${SHARE_PREFIX}/corr_modules"
+	cp -rf "${PWD}/corr_modules/"* "${SHARE_PREFIX}/corr_modules"
 	install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs"
 	install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs"
 	install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs"

README

@@ -30,7 +30,8 @@ Table of contents:
 	5. Correlation rules
 	6. Output database
 	7. Web interface
-	8. Additional documentation
+	8. Additional correlation modules
+	9. Additional documentation
 
 
 ===============================
@@ -161,18 +162,13 @@ following:
 
 preprocessor ai: \
 	alertfile "/your/snort/dir/log/alert" \
-	alert_history_file "/your/snort/dir/log/alert_history" \
-	alert_serialization_interval 3600 \
 	alert_bufsize 30 \
 	alert_clustering_interval 300 \
+	alert_correlation_weight 5000 \
+	alert_history_file "/your/snort/dir/log/alert_history" \
+	alert_serialization_interval 3600 \
 	bayesian_correlation_interval 1200 \
 	bayesian_correlation_cache_validity 600 \
-	correlation_graph_interval 300 \
-	correlation_rules_dir "/your/snort/dir/etc/corr_rules" \
-	correlated_alerts_dir "/your/snort/dir/log/correlated_alerts" \
-	correlation_threshold_coefficient 0.5 \
-	cluster_max_alert_interval 14400 \
-	clusterfile "/your/snort/dir/log/clustered_alerts" \
 	cluster ( class="dst_port", name="privileged_ports", range="1-1023" ) \
 	cluster ( class="dst_port", name="unprivileged_ports", range="1024-65535" ) \
 	cluster ( class="src_addr", name="local_net", range="192.168.1.0/24" ) \
@@ -181,10 +177,21 @@ preprocessor ai: \
 	cluster ( class="dst_addr", name="local_net", range="192.168.1.0/24" ) \
 	cluster ( class="dst_addr", name="dmz_net", range="155.185.0.0/16" ) \
 	cluster ( class="dst_addr", name="vpn_net", range="10.8.0.0/24" ) \
+	cluster_max_alert_interval 14400 \
+	clusterfile "/your/snort/dir/log/clustered_alerts" \
+	corr_modules_dir "/your/snort/dir/share/snort_ai_preproc/corr_modules" \
+	correlation_graph_interval 300 \
+	correlation_rules_dir "/your/snort/dir/etc/corr_rules" \
+	correlated_alerts_dir "/your/snort/dir/log/correlated_alerts" \
+	correlation_threshold_coefficient 0.5 \
 	database ( type="dbtype", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \
 	database_parsing_interval 30 \
 	hashtable_cleanup_interval 300 \
+	manual_correlations_parsing_interval 120 \
+	neural_network_training_interval 43200 \
+	neural_train_steps 10 \
 	output_database ( type="dbtype", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \
+	output_neurons_per_side 20 \
 	tcp_stream_expire_interval 300 \
 	webserv_banner "Snort AIPreprocessor module" \
 	webserv_dir "/prefix/share/htdocs" \
@@ -197,6 +204,15 @@ The options are the following:
 and   not  to  a  database  (default  if  not  specified:  /var/log/snort/alert)
 
 
+- alert_correlation_weight:  When this number of alert is stored in the "memory"
+of  the software (i.e. in the alert history file or in the output database), the
+weight  for  the  heuristical  correlation  indexes (bayesian network and neural
+ network)  will  be  more  or  less  equal  to  0.95,  on  a  scale from 0 to 1.
+This parameter expresses how much the heuristical indexes should be weighted and
+it  can  be  considered like a kind of "learning rate" for the alert correlation
+algorithm       (default       value      if      not      specified:      5000)
+
+
 - alert_history_file:  The   file keeping track of the history, in binary format,
 of   all  the  alerts  received  by  the  IDS, so that the module can build some
 statistical        correlation        inferences       over       the       past
@@ -234,6 +250,14 @@ in  the  bayesian  correlation  hash  table  (i.e.  a  pair  of  alerts with the
 before         being         updated        (default:        600        seconds)
 
 
+- corr_modules_dir:  This  software supports a kind of plugins, or "modules over
+the  module",  that  allow  the user to specify some extra correlation rules and
+indexes.  These  modules  are .so files placed in this directory (default if not
+ specified:  PREFIX/share/snort_ai_preproc/corr_modules),  dynamically loaded by
+the  module.  For  more  information  on  how  to write your own module, see the
+dedicated              section             in             this             file.
+
+
 - correlation_graph_interval:  The  interval that should occur from the building
 of  the correlation graph between the clustered alerts and the next one (default
  if              not             specified:             300             seconds)
@@ -301,6 +325,20 @@ the  hashtable  of  TCP  streams and the next one (default if not specified: 300
  seconds)
 
 
+- manual_correlations_parsing_interval: Interval in seconds between an execution
+of  the  thread for parsing the alert correlations manually set and the next one
+(default       value       if       not       specified:       120      seconds)
+
+
+- neural_network_training_interval:  Interval in seconds between an execution of
+the  thread  for  training the neural network using the set of recent alerts and
+the     next     one    (default    if    not    specified:    43200    seconds)
+
+
+- neural_train_steps:  Number  of  steps  to take in each training cycle for the
+neural                   network                  (default:                  10)
+
+
 - output_database:  Specify this option if you want to save the outputs from the
 module  (correlated  alerts,  clustered  alerts,  alerts  information  and their
 associated    packets   streams,   and  so  on)  to  a  relational  database  as
@@ -312,6 +350,13 @@ module,   just   give   the   right  file  to  your  database,  e.g.  for  MySQL
 $ mysql -uusername -ppassword dbname < schemas/mysql.sql
 
 
+- output_neurons_per_side: Number of output neurons per side on the output layer
+of  the  neural network (that is a rectangular matrix). A higher number allows a
+higher  granularity  over  similar  alerts, but a linear increment of this value
+produces  a  squared  increment of the computational complexity for the training
+and    evaluation    algorithms   (default   value   if   not   specified:   20)
+
+
 - tcp_stream_expire_interval:  The  interval that should occur for marking a TCP
 stream as "expired", if no more packets are received inside of that and it's not
 "marked"    as    suspicious   (default   if   not   specified:   300   seconds)
@@ -467,8 +512,41 @@ that,  all  the  alerts  of  those  types  will  be  marted  as  correlated,  or
 uncorrelated.
 
 
+=================================
+8. Additional correlation modules
+=================================
+
+It  is  possible  to  add  extra  parameters  and  indexes  for  evaluating  the
+correlation  between  two  alerts  in  an  extremely  simple  way. The directory
+specified  in  the  configuration  option  "corr_modules_dir" contains the extra
+modules  (as  binary  shared  libraries  ->  .so).  Each of these modules should
+contain          a          function          whose         prototype         is
+
+
+double  AI_corr_index ( AI_snort_alert*, AI_snort_alert* )
+
+
+taking  two alerts as parameters and returning a correlation value between them,
+and               one              whose              prototype              is
+
+
+double  AI_corr_index_weight ()
+
+
+returning  a  coefficient  in  [0,1]  expressing  the  weight  of that index. An
+example  module  is  contained  in  the  corr_modules  directory  in  the source
+directory,  or in PREFIX/share/snort_ai_preproc/corr_modules after installation.
+
+When  you  write  your  own module, just add in the Makefile in the corr_modules
+directory  a  line  like  the one already present there for compiling, then type
+`make'.   You   may   need   to   link   your   module  source  file(s)  against
+libsf_ai_preproc.la  if  you  want to use some of the functions from the module,
+ for  example,  for  reading  the  alerts  stored  in  the  history file, in the
+ database,       the       current       correlations,      and      so      on.
+
+
 ===========================
-8. Additional documentation
+9. Additional documentation
 ===========================
 
 The additional documentation over the code, functions and data structures can

TODO

@@ -2,10 +2,8 @@
 AVERAGE/HIGH PRIORITY:
 ======================
 
-- Supporting extra modules for alert correlation
 - Code profiling
 - Comment all the code!!!
-- Neural network for computing k
 - Testing more scenarios, making more hyperalert models
 
 =============
@@ -36,4 +34,5 @@ DONE:
 + Saving packet flows as .pcap
 + Manual alert correlation from the web interface
 + Neural network for alert correlation
++ Supporting extra modules for alert correlation
 

alert_parser.c

@@ -497,7 +497,7 @@ __AI_copy_alerts ( AI_snort_alert *node )
 AI_snort_alert*
 AI_get_alerts ()
 {
-	AI_snort_alert *alerts_copy;
+	AI_snort_alert *alerts_copy = NULL;
 
 	pthread_mutex_lock ( &alert_mutex );
 	alerts_copy = __AI_copy_alerts ( alerts );

cluster.c

@@ -751,7 +751,7 @@ __AI_copy_clustered_alerts ( AI_snort_alert *node )
 AI_snort_alert*
 AI_get_clustered_alerts ()
 {
-	AI_snort_alert *alerts_copy;
+	AI_snort_alert *alerts_copy = NULL;
 
 	pthread_mutex_lock ( &mutex );
 	alerts_copy = __AI_copy_clustered_alerts ( alert_log );

config.h.in

@@ -42,6 +42,9 @@
 /* Define to 1 if you have the `connect' function. */
 #undef HAVE_CONNECT
 
+/* Define to 1 if you have the <dirent.h> header file. */
+#undef HAVE_DIRENT_H
+
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #undef HAVE_DLFCN_H
 
@@ -60,6 +63,9 @@
 /* Define to 1 if you have the <inttypes.h> header file. */
 #undef HAVE_INTTYPES_H
 
+/* Define to 1 if you have the `dl' library (-ldl). */
+#undef HAVE_LIBDL
+
 /* Define to 1 if you have the `gvc' library (-lgvc). */
 #undef HAVE_LIBGVC
 

configure

@@ -11719,6 +11719,56 @@ as_fn_error $? "libm not found on the system
 See \`config.log' for more details" "$LINENO" 5 ; }
 fi
 
+{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5
+$as_echo_n "checking for dlopen in -ldl... " >&6; }
+if test "${ac_cv_lib_dl_dlopen+set}" = set; then :
+  $as_echo_n "(cached) " >&6
+else
+  ac_check_lib_save_LIBS=$LIBS
+LIBS="-ldl  $LIBS"
+cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+/* Override any GCC internal prototype to avoid an error.
+   Use char because int might match the return type of a GCC
+   builtin and then its argument prototype would still apply.  */
+#ifdef __cplusplus
+extern "C"
+#endif
+char dlopen ();
+int
+main ()
+{
+return dlopen ();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  ac_cv_lib_dl_dlopen=yes
+else
+  ac_cv_lib_dl_dlopen=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+LIBS=$ac_check_lib_save_LIBS
+fi
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5
+$as_echo "$ac_cv_lib_dl_dlopen" >&6; }
+if test "x$ac_cv_lib_dl_dlopen" = x""yes; then :
+  cat >>confdefs.h <<_ACEOF
+#define HAVE_LIBDL 1
+_ACEOF
+
+  LIBS="-ldl $LIBS"
+
+else
+  { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5
+$as_echo "$as_me: error: in \`$ac_pwd':" >&2;}
+as_fn_error $? "libdl not found on the system
+See \`config.log' for more details" "$LINENO" 5 ; }
+fi
+
 
 if test "x$prefix" == x/usr; then :
   CORR_RULES_PREFIX="/etc/snort/corr_rules"
@@ -11956,7 +12006,7 @@ _ACEOF
 
 fi
 
-for ac_header in inttypes.h limits.h stddef.h stdlib.h string.h unistd.h wchar.h math.h
+for ac_header in dirent.h dlfcn.h inttypes.h limits.h math.h stddef.h stdlib.h string.h unistd.h wchar.h
 do :
   as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh`
 ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default"

configure.ac

@@ -136,6 +136,7 @@ AS_IF([test "x$with_graphviz" != xno],
 AC_CHECK_LIB([xml2], [xmlReaderForFile],, AC_MSG_FAILURE(libxml2 not found on the system))
 AC_CHECK_LIB([pthread], [pthread_create],, AC_MSG_FAILURE(libpthread not found on the system))
 AC_CHECK_LIB([m], [sqrt],, AC_MSG_FAILURE(libm not found on the system))
+AC_CHECK_LIB([dl], [dlopen],, AC_MSG_FAILURE(libdl not found on the system))
 
 AS_IF([test "x$prefix" == x/usr],
 	[AC_SUBST([CORR_RULES_PREFIX], ["/etc/snort/corr_rules"])],
@@ -161,7 +162,7 @@ AS_IF([test "x$with_graphviz" != xno],
 	[AC_DEFINE([HAVE_BOOLEAN], [1], [Check if the boolean type is defined])])
 
 AC_FUNC_ALLOCA
-AC_CHECK_HEADERS([inttypes.h limits.h stddef.h stdlib.h string.h unistd.h wchar.h math.h],,AC_MSG_ERROR(At least one of the required headers was not found))
+AC_CHECK_HEADERS([dirent.h dlfcn.h inttypes.h limits.h math.h stddef.h stdlib.h string.h unistd.h wchar.h],,AC_MSG_ERROR(At least one of the required headers was not found))
 
 # Check for int types
 AC_CHECK_TYPES([u_int8_t,u_int16_t,u_int32_t,u_int64_t,uint8_t,uint16_t,uint32_t,uint64_t])

corr_modules/Makefile

@@ -0,0 +1,3 @@
+all:
+	gcc -I. -I.. -I../uthash -I../base64 -I../fsom -I../include -D_GNU_SOURCE -Wall -pedantic -pedantic-errors -std=c99 -fPIC -shared -rdynamic -o libsf_ai_corr_example.so libsf_ai_corr_example.c
+

corr_modules/libsf_ai_corr_example.c

@@ -0,0 +1,37 @@
+/*
+ * =====================================================================================
+ *
+ *       Filename:  libsf_ai_corr_example.c
+ *
+ *    Description:  Sample correlation module for two alerts
+ *
+ *        Version:  0.1
+ *        Created:  26/10/2010 14:23:24
+ *       Revision:  none
+ *       Compiler:  gcc
+ *
+ *         Author:  BlackLight (http://0x00.ath.cx), <blacklight@autistici.org>
+ *        Licence:  GNU GPL v.3
+ *        Company:  DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE!
+ *
+ * =====================================================================================
+ */
+
+#include	"spp_ai.h"
+
+/** Function that, given two alerts, returns a correlation index in [0,1] */
+
+double
+AI_corr_index ( const AI_snort_alert *a, const AI_snort_alert *b )
+{
+	return 0.5;
+}
+
+/** Function that returns the weight of this index */
+
+double
+AI_corr_index_weight ()
+{
+	return 0.0;
+}
+

correlation.c

@@ -102,9 +102,7 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp )
 		 src_port1[10],
 		 dst_port1[10],
 		 src_port2[10],
-		 dst_port2[10],
+		 dst_port2[10];
-		 *timestamp1,
-		 *timestamp2;
 
 	if ( !corr )
 		return;
@@ -115,37 +113,27 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp )
 	snprintf ( src_port1, sizeof ( src_port1 ), "%d", ntohs ( corr->key.a->tcp_src_port ));
 	snprintf ( dst_port1, sizeof ( dst_port1 ), "%d", ntohs ( corr->key.a->tcp_dst_port ));
 
-	timestamp1 = ctime ( &(corr->key.a->timestamp ) );
-	timestamp1 [ strlen ( timestamp1 ) - 1 ] = 0;
-
 	inet_ntop ( AF_INET, &(corr->key.b->ip_src_addr), src_addr2, INET_ADDRSTRLEN );
 	inet_ntop ( AF_INET, &(corr->key.b->ip_dst_addr), dst_addr2, INET_ADDRSTRLEN );
 
 	snprintf ( src_port2, sizeof ( src_port2 ), "%d", ntohs ( corr->key.b->tcp_src_port ));
 	snprintf ( dst_port2, sizeof ( dst_port2 ), "%d", ntohs ( corr->key.b->tcp_dst_port ));
 
-	timestamp2 = ctime ( &(corr->key.b->timestamp ) );
-	timestamp2 [ strlen ( timestamp2 ) - 1 ] = 0;
-
 	fprintf ( fp,
 		"\t\"[%d.%d.%d] %s\\n"
 		"%s:%s -> %s:%s\\n"
-		"%s\\n"
 		"(%d alerts grouped)\" -> "
 
 		"\"[%d.%d.%d] %s\\n"
 		"%s:%s -> %s:%s\\n"
-		"%s\\n"
 		"(%d alerts grouped)\";\n",
 
 		corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc,
 		src_addr1, src_port1, dst_addr1, dst_port1,
-		timestamp1,
 		corr->key.a->grouped_alerts_count,
 
 		corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc,
 		src_addr2, src_port2, dst_addr2, dst_port2,
-		timestamp2,
 		corr->key.b->grouped_alerts_count
 	);
 }		/* -----  end of function __AI_correlated_alerts_to_dot  ----- */
@@ -1230,7 +1218,10 @@ AI_alert_correlation_thread ( void *arg )
 						 bayesian_correlation  = 0.0,
 						 neural_correlation    = 0.0;
 
-	size_t                    n_correlations        = 0;
+	size_t                    n_correlations        = 0,
+						 n_corr_functions      = 0,
+						 n_corr_weights        = 0;
+
 	FILE                      *fp                   = NULL;
 
 	AI_alert_correlation_key  corr_key;
@@ -1255,6 +1246,12 @@ AI_alert_correlation_thread ( void *arg )
 	graph_t                   *g                    = NULL;
 	#endif
 
+	double (**corr_functions)( const AI_snort_alert*, const AI_snort_alert* ) = NULL;
+	double (**corr_weights)() = NULL;
+
+	corr_functions = AI_get_corr_functions( &n_corr_functions );
+	corr_weights   = AI_get_corr_weights ( &n_corr_weights );
+
 	pthread_mutex_init ( &mutex, NULL );
 
 	/* Start the thread for parsing manual correlations from XML */
@@ -1377,6 +1374,19 @@ AI_alert_correlation_thread ( void *arg )
 						n_correlations++;
 					}
 
+					/* Get the correlation indexes from extra correlation modules */
+					if (( corr_functions ))
+					{
+						for ( i=0; i < n_corr_functions; i++ )
+						{
+							if ( corr_weights[i]() != 0.0 )
+							{
+								corr->correlation += corr_weights[i]() * corr_functions[i] ( corr_key.a, corr_key.b );
+								n_correlations++;
+							}
+						}
+					}
+
 					if ( n_correlations != 0 )
 					{
 						corr->correlation /= (double) n_correlations;

modules.c

@@ -0,0 +1,151 @@
+/*
+ * =====================================================================================
+ *
+ *       Filename:  modules.c
+ *
+ *    Description:  Support for extra correlation modules
+ *
+ *        Version:  0.1
+ *        Created:  26/10/2010 01:11:25
+ *       Revision:  none
+ *       Compiler:  gcc
+ *
+ *         Author:  BlackLight (http://0x00.ath.cx), <blacklight@autistici.org>
+ *        Licence:  GNU GPL v.3
+ *        Company:  DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE!
+ *
+ * =====================================================================================
+ */
+
+#include	"spp_ai.h"
+
+#include	<dirent.h>
+#include	<dlfcn.h>
+#include	<stdlib.h>
+#include	<unistd.h>
+
+/** \defgroup modules Software component for loading extra user-provided modules for correlating alerts
+ * @{ */
+
+PRIVATE double (**corr_functions)(const AI_snort_alert*, const AI_snort_alert*) = NULL;
+PRIVATE size_t n_corr_functions = 0;
+
+PRIVATE double (**weight_functions)() = NULL;
+PRIVATE size_t n_weight_functions = 0;
+
+/**
+ * \brief  Get the correlation functions from the extra correlation modules as array of function pointers
+ * \param  n_functions 	Number of function pointers in the array
+ * \return The array of correlation functions
+ */
+
+double
+(**AI_get_corr_functions ( size_t *n_functions )) (const AI_snort_alert*, const AI_snort_alert*)
+{
+	*n_functions = n_corr_functions;
+	return corr_functions;
+}		/* -----  end of function AI_get_corr_functions  ----- */
+
+/**
+ * \brief  Get the weights of the correlation extra modules as array of function pointers
+ * \param  n_functions 	Number of function pointers in the array
+ * \return The array of correlation weights functions
+ */
+
+double
+(**AI_get_corr_weights ( size_t *n_functions )) ()
+{
+	*n_functions = n_weight_functions;
+	return weight_functions;
+}		/* -----  end of function AI_get_corr_weights  ----- */
+
+/**
+ * \brief  Initialize the extra modules provided by the user
+ */
+
+void
+AI_init_corr_modules ()
+{
+	void   **dl_handles = NULL;
+	DIR    *dir         = NULL;
+	char   *err         = NULL;
+	char   *fname       = NULL;
+	size_t n_dl_handles = 0;
+	struct dirent *dir_info = NULL;
+
+	if ( !( dir = opendir ( config->corr_modules_dir )))
+	{
+		return;
+	}
+
+	while (( dir_info = readdir ( dir )))
+	{
+		if ( preg_match ( "(\\.(l|s)o)|(\\.l?a)", dir_info->d_name, NULL, NULL ))
+		{
+			if ( !( dl_handles = (void**) realloc ( dl_handles, (++n_dl_handles) * sizeof ( void* ))))
+			{
+				AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
+			}
+
+			if ( !( fname = (char*) malloc ( strlen ( config->corr_modules_dir ) + strlen ( dir_info->d_name ) + 4 )))
+			{
+				AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
+			}
+
+			sprintf ( fname, "%s/%s", config->corr_modules_dir, dir_info->d_name );
+
+			if ( !( dl_handles[n_dl_handles-1] = dlopen ( fname, RTLD_LAZY )))
+			{
+				if (( err = dlerror() ))
+				{
+					_dpd.errMsg ( "dlopen: %s\n", err );
+				}
+
+				AI_fatal_err ( "dlopen error", __FILE__, __LINE__ );
+			}
+
+			free ( fname );
+			fname = NULL;
+
+			if ( !( corr_functions = (double(**)(const AI_snort_alert*, const AI_snort_alert*))
+						realloc ( corr_functions, (++n_corr_functions) * sizeof ( double(*)(const AI_snort_alert*, const AI_snort_alert*) ))))
+			{
+				AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
+			}
+
+			*(void**) (&(corr_functions[ n_corr_functions - 1 ])) = dlsym ( dl_handles[n_dl_handles-1], "AI_corr_index" );
+
+			if ( !corr_functions[ n_corr_functions - 1 ] )
+			{
+				if (( err = dlerror() ))
+				{
+					_dpd.errMsg ( "dlsym: %s\n", err );
+				}
+
+				AI_fatal_err ( "dlsym error", __FILE__, __LINE__ );
+			}
+
+			if ( !( weight_functions = (double(**)()) realloc ( weight_functions, (++n_weight_functions) * sizeof ( double(*)() ))))
+			{
+				AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
+			}
+
+			*(void**) (&(weight_functions[ n_weight_functions - 1 ])) = dlsym ( dl_handles[n_dl_handles-1], "AI_corr_index_weight" );
+
+			if ( !weight_functions[ n_weight_functions - 1 ] )
+			{
+				if (( err = dlerror() ))
+				{
+					_dpd.errMsg ( "dlsym: %s\n", err );
+				}
+
+				AI_fatal_err ( "dlsym error", __FILE__, __LINE__ );
+			}
+		}
+	}
+
+	closedir ( dir );
+}		/* -----  end of function AI_init_corr_modules  ----- */
+
+/** @} */
+

neural.c

@@ -251,7 +251,7 @@ __AI_som_train ()
 
 	if ( !( res = (DB_result) DB_out_query ( query )))
 	{
-		AI_fatal_err ( "AIPreproc: Query error", __FILE__, __LINE__ );
+		return;
 	}
 
 	num_rows = DB_num_rows ( res );
@@ -277,7 +277,7 @@ __AI_som_train ()
 	{
 		row = (DB_row) DB_fetch_row ( res );
 
-		tuples[i].gid = row[0] ? strtoul ( row[0], NULL, 10 ) : 0;
+		
 		tuples[i].sid = row[1] ? strtoul ( row[1], NULL, 10 ) : 0;
 		tuples[i].rev = row[2] ? strtoul ( row[2], NULL, 10 ) : 0;
 		tuples[i].timestamp = row[3] ? (time_t) strtol ( row[3], NULL, 10 ) : (time_t) 0;

spp_ai.c

@@ -110,6 +110,9 @@ static void AI_init(char *args)
 	sfPolicyUserPolicySet(ex_config, policy_id);
 	sfPolicyUserDataSetCurrent(ex_config, config);
 
+	/* Initialize the extra correlation modules */
+	AI_init_corr_modules();
+
 	/* If the hash_cleanup_interval or stream_expire_interval options are set to zero,
 	 * no cleanup will be made on the streams */
 	if ( config->hashCleanupInterval != 0 && config->streamExpireInterval != 0 )
@@ -173,8 +176,9 @@ static AI_config * AI_parse(char *args)
 	char alertfile[1024]          = { 0 },
 		alert_history_file[1024] = { 0 },
 		clusterfile[1024]        = { 0 },
-		corr_rules_dir[1024]     = { 0 },
 		corr_alerts_dir[1024]    = { 0 },
+		corr_modules_dir[1024]   = { 0 },
+		corr_rules_dir[1024]     = { 0 },
 		webserv_dir[1024]        = { 0 },
 		webserv_banner[1024]     = { 0 };
 
@@ -209,6 +213,7 @@ static AI_config * AI_parse(char *args)
 			     clusterfile_len                      = 0,
 			     cluster_max_alert_interval           = 0,
 			     corr_alerts_dir_len                  = 0,
+				corr_modules_dir_len                 = 0,
 			     corr_rules_dir_len                   = 0,
 			     correlation_graph_interval           = 0,
 			     database_parsing_interval            = 0,
@@ -224,6 +229,7 @@ static AI_config * AI_parse(char *args)
 		has_stream_expire_interval  = false,
 		has_correlation_interval    = false,
 		has_corr_alerts_dir         = false,
+		has_corr_modules_dir        = false,
 		has_database_interval       = false,
 		has_webserv_dir             = false,
 		has_webserv_banner          = false,
@@ -791,6 +797,46 @@ static AI_config * AI_parse(char *args)
 
 	_dpd.logMsg("    webserv_dir: %s\n", config->webserv_dir);
 
+	/* Parsing the corr_modules_dir option */
+	if (( arg = (char*) strcasestr( args, "corr_modules_dir" ) ))
+	{
+		for ( arg += strlen("corr_modules_dir");
+				*arg && *arg != '"';
+				arg++ );
+
+		if ( !(*(arg++)) )
+		{
+			AI_fatal_err ( "corr_modules_dir option used but no filename specified", __FILE__, __LINE__ );
+		}
+
+		for ( corr_modules_dir[ (++corr_modules_dir_len)-1 ] = *arg;
+				*arg && *arg != '"' && corr_modules_dir_len < sizeof ( corr_modules_dir );
+				arg++, corr_modules_dir[ (++corr_modules_dir_len)-1 ] = *arg );
+
+		if ( corr_modules_dir[0] == 0 || corr_modules_dir_len <= 1 )  {
+			has_corr_modules_dir = false;
+		} else {
+			if ( corr_modules_dir_len >= sizeof ( corr_modules_dir ))  {
+				AI_fatal_err ( "corr_modules_dir path too long ( >= 1024 )", __FILE__, __LINE__ );
+			} else if ( strlen( corr_modules_dir ) == 0 ) {
+				has_corr_modules_dir = false;
+			} else {
+				has_corr_modules_dir = true;
+				corr_modules_dir[ corr_modules_dir_len-1 ] = 0;
+				strncpy ( config->corr_modules_dir, corr_modules_dir, corr_modules_dir_len );
+			}
+		}
+	}
+
+	if ( ! has_corr_modules_dir )
+	{
+		#ifndef HAVE_CONFIG_H
+			AI_fatal_err ( "Unable to read PREFIX from config.h", __FILE__, __LINE__  );
+		#endif
+
+		snprintf ( config->corr_modules_dir, sizeof ( config->corr_modules_dir ), "%s/share/snort_ai_preprocessor/corr_modules", PREFIX );
+	}
+
 	/* Neural network output file */
 	if ( config->neuralNetworkTrainingInterval != 0 )
 	{

spp_ai.h

@@ -94,7 +94,7 @@
 /** Default number of alerts needed in the history file or database for letting a certain
  * heuristic correlation index weight be =~ 0.95 (the weight monotonically increases
  * with the number of alerts according to a hyperbolic tangent function) */
-#define 	DEFAULT_ALERT_CORRELATION_WEIGHT 		400
+#define 	DEFAULT_ALERT_CORRELATION_WEIGHT 		5000
 
 /** Default web server port */
 #define 	DEFAULT_WEBSERV_PORT 				7654
@@ -105,7 +105,7 @@
 /** Cutoff y value in the exponential decay for considering two alerts not correlated */
 #define 	CUTOFF_Y_VALUE 					0.01
 
-/** Approximated solution of the equation tanh(x) = 0.95 */
+/** Approximated solution of the equation tanh(x) = 0.95 (used as parameter in the correlation indexes weight function) */
 #define 	HYPERBOLIC_TANGENT_SOLUTION 	1.83178
 
 /****************************/
@@ -233,6 +233,9 @@ typedef struct
 	 * and in the footer of error pages */
 	char          webserv_banner[1024];
 
+	/** Directory containing extra correlation modules */
+	char          corr_modules_dir[1024];
+
 	/** Alert file */
 	char          alertfile[1024];
 
@@ -486,6 +489,7 @@ void               AI_pkt_enqueue ( SFSnortPacket* );
 void               AI_set_stream_observed ( struct pkt_key key );
 void               AI_hierarchies_build ( hierarchy_node**, int );
 void               AI_free_alerts ( AI_snort_alert *node );
+void               AI_init_corr_modules ();
 
 struct pkt_info*   AI_get_stream_by_key ( struct pkt_key );
 AI_snort_alert*    AI_get_alerts ( void );
@@ -508,6 +512,9 @@ void*                  AI_store_alert_to_db_thread ( void* );
 void*                  AI_store_cluster_to_db_thread ( void* );
 void*                  AI_store_correlation_to_db_thread ( void* );
 
+double(**AI_get_corr_functions ( size_t* ))(const AI_snort_alert*, const AI_snort_alert*);
+double(**AI_get_corr_weights ( size_t* ))();
+
 /** Function pointer to the function used for getting the alert list (from log file, db, ...) */
 extern AI_snort_alert* (*get_alerts)(void);