Improved multithread locks management

README

@@ -147,7 +147,8 @@ following:
 preprocessor ai: \
 	hashtable_cleanup_interval 300 \
 	tcp_stream_expire_interval 300 \
-	alertfile "/home/youruser/local/snort/log/alert" \
+	alertfile "/your/snort/dir/log/alert" \
+	alert_history_file "/your/snort/dir/log/alert_history" \
 	alert_clustering_interval 300 \
 	correlation_graph_interval 300 \
 	correlation_rules_dir "/your/snort/dir/etc/corr_rules" \
@@ -178,6 +179,10 @@ stream as "expired", if no more packets are received inside of that and it's not
 - alertfile:  The file where Snort saves its alerts, if they are saved to a file
 and   not  to  a  database  (default  if  not  specified:  /var/log/snort/alert)
 
+- alert_history_file:  The   file keeping track of the history, in binary format,
+of   all  the  alerts  received  by  the  IDS, so that the module can build some
+statistical        correlation        inferences       over       the       past
+
 - alert_clustering_interval:  The interval that should occur from the clustering
 of  the  alerts  in the log according to the provided clustering hierarchies and
 the     next     one     (default     if    not    specified:    300    seconds)

TODO

@@ -2,7 +2,6 @@
 AVERAGE/HIGH PRIORITY:
 ======================
 
-- Dynamic k parameter in correlation threshold
 - Testing more scenarios, making more hyperalert models
 - Bayesian learning among alerts in alert log
 - libgc support

alert_history.c

@@ -0,0 +1,21 @@
+/*
+ * =====================================================================================
+ *
+ *       Filename:  alert_history.c
+ *
+ *    Description:  Manages the history of alerts on a binary file
+ *
+ *        Version:  0.1
+ *        Created:  18/09/2010 21:02:15
+ *       Revision:  none
+ *       Compiler:  gcc
+ *
+ *         Author:  BlackLight (http://0x00.ath.cx), <blacklight@autistici.org>
+ *        Licence:  GNU GPL v.3
+ *        Company:  DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE!
+ *
+ * =====================================================================================
+ */
+
+#include	"spp_ai.h"
+

alert_parser.c

@@ -29,9 +29,9 @@
 #include 	<pthread.h>
 
 
-PRIVATE AI_snort_alert *alerts   = NULL;
+PRIVATE AI_snort_alert   *alerts   = NULL;
-PRIVATE FILE           *alert_fp = NULL;
+PRIVATE FILE             *alert_fp = NULL;
-PRIVATE BOOL           lock_flag = false;
+PRIVATE pthread_mutex_t  mutex;
 
 /** \defgroup alert_parser Parse the alert log into binary structures
  * @{ */
@@ -78,6 +78,8 @@ AI_file_alertparser_thread ( void* arg )
 	AI_snort_alert *tmp    = NULL;
 	BOOL in_alert          = false;
 
+	pthread_mutex_init ( &mutex, NULL );
+
 	while ( 1 )
 	{
 #ifndef MACOS
@@ -132,6 +134,7 @@ AI_file_alertparser_thread ( void* arg )
 				fd = fileno(alert_fp);
 			}
 		}
+
 		/*
 		 * Cause the thread to wait until a new file modification (a new alert).
 		 */
@@ -139,6 +142,7 @@ AI_file_alertparser_thread ( void* arg )
 			usleep(100);
 			fstats( fd, &stats );
 		}
+
 		/*
 		 * The first time the thread is called, the flow exits instantly from the while,
 		 * so this first time the stats structure has to be initialized properly.
@@ -153,7 +157,6 @@ AI_file_alertparser_thread ( void* arg )
 #endif
 		
 		/* Set the lock flag to true until it's done with alert parsing */
-		lock_flag = true;
 
 		while ( !feof ( alert_fp ))
 		{
@@ -205,6 +208,8 @@ AI_file_alertparser_thread ( void* arg )
 
 			if ( !in_alert )
 			{
+				pthread_mutex_lock ( &mutex );
+
 				if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 )
 				{
 					in_alert = true;
@@ -351,9 +356,11 @@ AI_file_alertparser_thread ( void* arg )
 			}
 		}
 
-		lock_flag = false;
+		pthread_mutex_unlock ( &mutex );
+		/* AI_alert_serialize ( alert, conf ); */
 	}
 
+	pthread_mutex_destroy ( &mutex );
 	pthread_exit ((void*) 0 );
 	return (void*) 0;
 }		/* -----  end of function AI_file_alertparser_thread  ----- */
@@ -397,8 +404,13 @@ _AI_copy_alerts ( AI_snort_alert *node )
 AI_snort_alert*
 AI_get_alerts ()
 {
-	while ( lock_flag );
+	AI_snort_alert *alerts_copy;
-	return _AI_copy_alerts ( alerts );
+
+	pthread_mutex_lock ( &mutex );
+	alerts_copy = _AI_copy_alerts ( alerts );
+	pthread_mutex_unlock ( &mutex );
+
+	return alerts_copy;
 }		/* -----  end of function AI_get_alerts  ----- */
 
 

cluster.c

@@ -50,11 +50,10 @@ typedef struct  {
 } AI_alert_occurrence;
 
 
-PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL };
+PRIVATE hierarchy_node  *h_root[CLUSTER_TYPES] = { NULL };
-PRIVATE AI_config      *_config               = NULL;
+PRIVATE AI_config       *_config               = NULL;
-PRIVATE AI_snort_alert *alert_log             = NULL;
+PRIVATE AI_snort_alert  *alert_log             = NULL;
-PRIVATE BOOL           lock_flag              = false;
+PRIVATE pthread_mutex_t  mutex;
-
 
 /**
  * \brief  Function that picks up the heuristic value for a clustering attribute in according to Julisch's heuristic (ACM, Vol.2, No.3, 09 2002, pag.124)
@@ -373,7 +372,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
 {
 	AI_snort_alert *tmp;
 	char ip[INET_ADDRSTRLEN];
-	char *timestamp;
+	char timestamp[128];
+	struct tm *_tm;
 
 	for ( tmp = log; tmp; tmp = tmp->next )
 	{
@@ -384,8 +384,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
 
 		fprintf ( fp, "[Priority: %d]\n", tmp->priority );
 
-		timestamp = ctime ( &tmp->timestamp );
+		_tm = localtime ( &tmp->timestamp );
-		timestamp[ strlen(timestamp)-1 ] = 0;
+		strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm );
 		fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
 
 		if ( h_root[src_addr] && tmp->h_node[src_addr] )
@@ -445,13 +445,15 @@ _AI_cluster_thread ( void* arg )
 	int            single_alerts_count = 0;
 	double         heterogeneity = 0;
 
+	pthread_mutex_init ( &mutex, NULL );
+
 	while ( 1 )
 	{
 		/* Between an execution of the thread and the next one, sleep for alert_clustering_interval seconds */
 		sleep ( _config->alertClusteringInterval );
 
 		/* Set the lock over the alert log until it's done with the clustering operation */
-		lock_flag = true;
+		pthread_mutex_lock ( &mutex );
 
 		/* Free the current alert log and get the latest one */
 		if ( alert_log )
@@ -462,7 +464,7 @@ _AI_cluster_thread ( void* arg )
 		
 		if ( !( alert_log = get_alerts() ))
 		{
-			lock_flag = false;
+			pthread_mutex_unlock ( &mutex );
 			continue;
 		}
 
@@ -561,7 +563,7 @@ _AI_cluster_thread ( void* arg )
 			alert_count -= _AI_merge_alerts ( &alert_log );
 		} while ( old_alert_count != alert_count );
 
-		lock_flag = false;
+		pthread_mutex_unlock ( &mutex );
 
 		if ( !( cluster_fp = fopen ( _config->clusterfile, "w" )) )
 		{
@@ -732,8 +734,13 @@ _AI_copy_clustered_alerts ( AI_snort_alert *node )
 AI_snort_alert*
 AI_get_clustered_alerts ()
 {
-	for ( ; lock_flag; usleep(100) );
+	AI_snort_alert *alerts_copy;
-	return _AI_copy_clustered_alerts ( alert_log );
+
+	pthread_mutex_lock ( &mutex );
+	alerts_copy = _AI_copy_clustered_alerts ( alert_log );
+	pthread_mutex_unlock ( &mutex );
+
+	return alerts_copy;
 }		/* -----  end of function AI_get_clustered_alerts  ----- */
 
 /** @} */

correlation.c

@@ -70,7 +70,7 @@ PRIVATE AI_hyperalert_info   *hyperalerts       = NULL;
 PRIVATE AI_config            *conf              = NULL;
 PRIVATE AI_snort_alert       *alerts            = NULL;
 PRIVATE AI_alert_correlation *correlation_table = NULL;
-PRIVATE BOOL                 lock_flag          = false;
+PRIVATE pthread_mutex_t      mutex;
 
 /**
  * \brief  Clean up the correlation hash table
@@ -713,6 +713,7 @@ AI_alert_correlation_thread ( void *arg )
 	#endif
 
 	conf = (AI_config*) arg;
+	pthread_mutex_init ( &mutex, NULL );
 
 	while ( 1 )
 	{
@@ -727,7 +728,7 @@ AI_alert_correlation_thread ( void *arg )
 		}
 
 		/* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */
-		lock_flag = true;
+		pthread_mutex_lock ( &mutex );
 
 		if ( alerts )
 		{
@@ -737,7 +738,7 @@ AI_alert_correlation_thread ( void *arg )
 
 		if ( !( alerts = AI_get_clustered_alerts() ))
 		{
-			lock_flag = false;
+			pthread_mutex_unlock ( &mutex );
 			continue;
 		}
 
@@ -894,7 +895,7 @@ AI_alert_correlation_thread ( void *arg )
 			#endif
 		}
 
-		lock_flag = false;
+		pthread_mutex_unlock ( &mutex );
 	}
 
 	pthread_exit (( void* ) 0 );

db.c

@@ -30,12 +30,9 @@
  * @{ */
 
 
-PRIVATE AI_config      *config;
+PRIVATE AI_config        *config;
-PRIVATE AI_snort_alert *alerts   = NULL;
+PRIVATE AI_snort_alert   *alerts   = NULL;
-PRIVATE BOOL           lock_flag = false;
+PRIVATE pthread_mutex_t  mutex;
-
-/** pthread mutex for accessing database data */
-PRIVATE pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER;
 
 /**
  * \brief  Thread for parsing alerts from a database
@@ -65,7 +62,7 @@ AI_db_alertparser_thread ( void *arg )
 	}
 
 	config = ( AI_config* ) arg;
-	pthread_mutex_lock ( &db_mutex );
+	pthread_mutex_init ( &mutex, NULL );
 
 	if ( !DB_init ( config ))
 	{
@@ -73,14 +70,10 @@ AI_db_alertparser_thread ( void *arg )
 				config->dbname, config->dbhost );
 	}
 
-	pthread_mutex_unlock ( &db_mutex );
-
 	while ( 1 )
 	{
 		sleep ( config->databaseParsingInterval );
-
+		pthread_mutex_lock ( &mutex );
-		/* Set the lock flag to true until it's done with alert parsing */
-		lock_flag = true;
 
 		memset ( query, 0, sizeof ( query ));
 		snprintf ( query, sizeof (query), "select cid, unix_timestamp(timestamp), signature from event where cid > %d "
@@ -98,7 +91,7 @@ AI_db_alertparser_thread ( void *arg )
 			DB_close();
 			_dpd.fatalMsg ( "AIPreproc: Could not store the query result at %s:%d\n", __FILE__, __LINE__ );
 		} else if ( rows == 0 ) {
-			lock_flag = false;
+			pthread_mutex_unlock ( &mutex );
 			continue;
 		}
 
@@ -224,7 +217,7 @@ AI_db_alertparser_thread ( void *arg )
 			}
 		}
 		
-		lock_flag = false;
+		pthread_mutex_unlock ( &mutex );
 		DB_free_result ( res );
 		latest_time = time ( NULL );
 	}
@@ -272,8 +265,13 @@ _AI_db_copy_alerts ( AI_snort_alert *node )
 AI_snort_alert*
 AI_db_get_alerts ()
 {
-	while ( lock_flag );
+	AI_snort_alert *alerts_copy;
-	return _AI_db_copy_alerts ( alerts );
+
+	pthread_mutex_lock ( &mutex );
+	alerts_copy = _AI_db_copy_alerts ( alerts );
+	pthread_mutex_unlock ( &mutex );
+
+	return alerts_copy;
 }		/* -----  end of function AI_db_get_alerts  ----- */
 
 /** @} */

spp_ai.c

@@ -135,10 +135,11 @@ static AI_config * AI_parse(char *args)
 {
 	char *arg;
 	char *match;
-	char alertfile[1024]       = { 0 };
+	char alertfile[1024]          = { 0 };
-	char clusterfile[1024]     = { 0 };
+	char clusterfile[1024]        = { 0 };
-	char corr_rules_dir[1024]  = { 0 };
+	char corr_rules_dir[1024]     = { 0 };
-	char corr_alerts_dir[1024] = { 0 };
+	char corr_alerts_dir[1024]    = { 0 };
+	char alert_history_file[1024] = { 0 };
 
 	char **matches       = NULL;
 	int  nmatches        = 0;
@@ -160,6 +161,7 @@ static AI_config * AI_parse(char *args)
 	unsigned long cleanup_interval           = 0,
 			    stream_expire_interval     = 0,
 			    alertfile_len              = 0,
+			    alert_history_file_len     = 0,
 			    clusterfile_len            = 0,
 			    corr_rules_dir_len         = 0,
 			    corr_alerts_dir_len        = 0,
@@ -176,7 +178,8 @@ static AI_config * AI_parse(char *args)
 		has_clusterfile             = false,
 		has_corr_rules_dir          = false,
 		has_clustering              = false,
-		has_database_log            = false;
+		has_database_log            = false,
+		has_alert_history_file      = false;
 
 	AI_config *config                = NULL;
 
@@ -336,6 +339,38 @@ static AI_config * AI_parse(char *args)
 		}
 	}
 
+	/* Parsing the alert_history_file option */
+	if (( arg = (char*) strcasestr( args, "alert_history_file" ) ))
+	{
+		for ( arg += strlen("alert_history_file");
+				*arg && *arg != '"';
+				arg++ );
+
+		if ( !(*(arg++)) )
+		{
+			_dpd.fatalMsg("AIPreproc: alert_history_file option used but no filename specified\n");
+		}
+
+		for ( alert_history_file[ (++alert_history_file_len)-1 ] = *arg;
+				*arg && *arg != '"' && alert_history_file_len < 1024;
+				arg++, alert_history_file[ (++alert_history_file_len)-1 ] = *arg );
+
+		if ( alert_history_file[0] == 0 || alert_history_file_len <= 1 )  {
+			has_alert_history_file = false;
+		} else {
+			if ( alert_history_file_len >= 1024 )  {
+				_dpd.fatalMsg("AIPreproc: alert_history_file path too long ( >= 1024 )\n");
+			} else if ( strlen( alert_history_file ) == 0 ) {
+				has_alert_history_file = false;
+			} else {
+				has_alert_history_file = true;
+				alert_history_file [ alert_history_file_len-1 ] = 0;
+				strncpy ( config->alert_history_file, alert_history_file, alert_history_file_len );
+				_dpd.logMsg("    alert_history_file path: %s\n", config->alert_history_file);
+			}
+		}
+	}
+
 	/* Parsing the clusterfile option */
 	if (( arg = (char*) strcasestr( args, "clusterfile" ) ))
 	{
@@ -767,6 +802,12 @@ static AI_config * AI_parse(char *args)
 		alertparser_thread = AI_file_alertparser_thread;
 	}
 
+	if ( !has_alert_history_file )
+	{
+		strncpy ( config->alert_history_file, DEFAULT_ALERT_HISTORY_FILE, sizeof ( config->alert_history_file ));
+		has_alert_history_file = true;
+	}
+
 	if ( has_clustering )
 	{
 		if ( ! hierarchy_nodes )

spp_ai.h

@@ -57,6 +57,9 @@
 /** Default directory for placing correlated alerts information (.dot and possibly .png files) */
 #define 	DEFAULT_CORR_ALERTS_DIR 				"/var/log/snort/correlated_alerts"
 
+/** Default path to alert history binary file, used for bayesian statistical correlation over alerts */
+#define 	DEFAULT_ALERT_HISTORY_FILE 			"/var/log/snort/alert_history"
+
 /** Default correlation threshold coefficient for correlating two hyperalerts */
 #define 	DEFAULT_CORR_THRESHOLD 				0.5
 
@@ -144,6 +147,9 @@ typedef struct
 	/** Alert file */
 	char          alertfile[1024];
 
+	/** Alert history binary file */
+	char          alert_history_file[1024];
+
 	/** Clustered alerts file */
 	char          clusterfile[1024];
 
@@ -305,6 +311,9 @@ struct pkt_info*   AI_get_stream_by_key ( struct pkt_key );
 AI_snort_alert*    AI_get_alerts ( void );
 AI_snort_alert*    AI_get_clustered_alerts ( void );
 
+void               AI_serialize_alert ( AI_snort_alert*, AI_config* );
+void               AI_deserialize_alert ( AI_snort_alert*, AI_config* );
+
 /** Function pointer to the function used for getting the alert list (from log file, db, ...) */
 extern AI_snort_alert* (*get_alerts)(void);