Grouped alert info inside the cluster, better graph management in hyperalerts

TODO

@@ -1,9 +1,25 @@
+======================
+AVERAGE/HIGH PRIORITY:
+======================
+
+- Dynamic cluster_min_size algorithm
+- Dynamic k parameter in correlation threshold
+- Testing more scenarios, making more hyperalert models
 - Bayesian learning among alerts in alert log
 - libgc support
-+ PostgreSQL support [DONE]
+
-- Dynamic k parameter in correlation threshold
+=============
-- Dynamic cluster_min_size algorithm
+LOW PRIORITY:
-+ Regex comp cache [DONE]
+=============
 
 - Managing clusters for addresses, timestamps (and more?)
 
+=====
+DONE:
+=====
+
++ PostgreSQL support
++ Regex comp cache
++ Managing hyperalert graph connection inside the alert structure itself
++ Keeping track of all the streams and alerts even after clustered
+

alert_parser.c

@@ -417,6 +417,20 @@ AI_free_alerts ( AI_snort_alert *node )
 	if ( node->next )
 		AI_free_alerts ( node->next );
 
+	/* if ( node->grouped_alerts ) */
+	/* { */
+	/* 	for ( i=0; i < node->grouped_alerts_count; i++ ) */
+	/* 	{ */
+	/* 		if ( node->grouped_alerts[i] ) */
+	/* 		{ */
+	/* 			free ( node->grouped_alerts[i] ); */
+	/* 			node->grouped_alerts[i] = NULL; */
+	/* 		} */
+	/* 	} */
+
+	/* 	free ( node->grouped_alerts ); */
+	/* } */
+
 	if ( node->hyperalert )
 	{
 		for ( i=0; i < node->hyperalert->n_preconds; i++ )
@@ -431,6 +445,9 @@ AI_free_alerts ( AI_snort_alert *node )
 		node->hyperalert = NULL;
 	}
 
+	if ( node->parent_alerts )
+		free ( node->parent_alerts );
+
 	if ( node->derived_alerts )
 		free ( node->derived_alerts );
 

cluster.c

@@ -268,12 +268,15 @@ _AI_merge_alerts ( AI_snort_alert **log )
 				{
 					if ( _AI_equal_alarms ( tmp, tmp2->next ))
 					{
-						tmp3 = tmp2->next->next;
+						if ( !( tmp->grouped_alerts = ( AI_snort_alert** ) realloc ( tmp->grouped_alerts, (++(tmp->grouped_alerts_count)) * sizeof ( AI_snort_alert* ))))
-						free ( tmp2->next );
+							_dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
-						tmp2->next = tmp3;
 
-						tmp->grouped_alarms_count++;
+						tmp->grouped_alerts[ tmp->grouped_alerts_count - 1 ] = tmp2->next;
 						count++;
+
+						tmp3 = tmp2->next->next;
+						/* free ( tmp2->next ); */
+						tmp2->next = tmp3;
 					}
 				}
 
@@ -311,7 +314,7 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
 
 		timestamp = ctime ( &tmp->timestamp );
 		timestamp[ strlen(timestamp)-1 ] = 0;
-		fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alarms_count, timestamp );
+		fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
 
 		if ( h_root[src_addr] && tmp->h_node[src_addr] )
 		{
@@ -395,13 +398,13 @@ _AI_cluster_thread ( void* arg )
 		for ( tmp = alert_log, alert_count=0; tmp; tmp = tmp->next, alert_count++ )
 		{
 			/* If an alert has an unitialized "grouped alarms count", set its counter to 1 (it only groupes the current alert) */
-			if ( tmp->grouped_alarms_count == 0 )
+			if ( tmp->grouped_alerts_count == 0 )
 			{
-				tmp->grouped_alarms_count = 1;
+				tmp->grouped_alerts_count = 1;
 			}
 
 			/* If the current alarm already group at least min_size alarms, then no need to do further clusterization */
-			if ( tmp->grouped_alarms_count >= cluster_min_size )
+			if ( tmp->grouped_alerts_count >= cluster_min_size )
 			{
 				has_small_clusters = false;
 			}
@@ -475,7 +478,7 @@ _AI_cluster_thread ( void* arg )
 			/* For all the alerts, the corresponing clustering value is the parent of the current one in the hierarchy */
 			for ( tmp = alert_log; tmp; tmp = tmp->next )
 			{
-				if ( tmp->grouped_alarms_count < cluster_min_size && tmp->h_node[best_type] )
+				if ( tmp->grouped_alerts_count < cluster_min_size && tmp->h_node[best_type] )
 				{
 					if ( tmp->h_node[best_type]->parent )
 					{

correlation.c

@@ -147,12 +147,12 @@ _AI_print_correlated_alerts ( AI_alert_correlation *corr, FILE *fp, BOOL strong
 		corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc,
 		src_addr1, src_port1, dst_addr1, dst_port1,
 		timestamp1,
-		corr->key.a->grouped_alarms_count,
+		corr->key.a->grouped_alerts_count,
 
 		corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc,
 		src_addr2, src_port2, dst_addr2, dst_port2,
 		timestamp2,
-		corr->key.b->grouped_alarms_count,
+		corr->key.b->grouped_alerts_count,
 		strong ? "" : "[style=dotted]"
 	);
 }		/* -----  end of function _AI_correlation_flow_to_file  ----- */
@@ -687,7 +687,8 @@ AI_alert_correlation_thread ( void *arg )
 {
 	int                       i;
 	struct stat               st;
-	char                      corr_dot_file[4096]   = { 0 };
+	char                      corr_dot_file[4096]   = { 0 },
+						 corr_ps_file [4096]   = { 0 };
 
 	double                    avg_correlation       = 0.0,
 						 std_deviation         = 0.0,
@@ -858,8 +859,11 @@ AI_alert_correlation_thread ( void *arg )
 					if ( !( corr->key.a->derived_alerts = ( AI_snort_alert** ) realloc ( corr->key.a->derived_alerts, (++corr->key.a->n_derived_alerts) * sizeof ( AI_snort_alert* ))))
 						_dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ );
 
+					if ( !( corr->key.b->parent_alerts = ( AI_snort_alert** ) realloc ( corr->key.b->parent_alerts, (++corr->key.b->n_parent_alerts) * sizeof ( AI_snort_alert* ))))
+						_dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ );
+
 					corr->key.a->derived_alerts[ corr->key.a->n_derived_alerts - 1 ] = corr->key.b;
-					corr->key.b->previous_correlated = corr->key.a;
+					corr->key.b->parent_alerts [ corr->key.b->n_parent_alerts  - 1 ] = corr->key.a;
 					_AI_print_correlated_alerts ( corr, fp, ( corr->correlation >= corr_strong_threshold ));
 				}
 			}
@@ -869,6 +873,7 @@ AI_alert_correlation_thread ( void *arg )
 
 			#ifdef HAVE_LIBGVC
 				snprintf ( corr_png_file, sizeof ( corr_png_file ), "%s/correlated_alerts.png", conf->corr_alerts_dir );
+				snprintf ( corr_ps_file , sizeof ( corr_ps_file  ), "%s/correlated_alerts.ps" , conf->corr_alerts_dir );
 
 				if ( !( gvc = gvContext() ))
 					continue;
@@ -881,6 +886,7 @@ AI_alert_correlation_thread ( void *arg )
 
 				gvLayout ( gvc, g, "dot" );
 				gvRenderFilename ( gvc, g, "png", corr_png_file );
+				gvRenderFilename ( gvc, g, "ps" , corr_ps_file  );
 
 				gvFreeLayout ( gvc, g );
 				agclose ( g );

spp_ai.h

@@ -253,18 +253,25 @@ typedef struct _AI_snort_alert  {
 	 * if the clustering algorithm is used */
 	hierarchy_node  *h_node[CLUSTER_TYPES];
 
+	/** If the clustering algorithm is used,
+	 * keep tracked of the pointers to the
+	 * single grouped alerts */
+	struct _AI_snort_alert **grouped_alerts;
+
 	/** If the clustering algorithm is used,
 	 * we also count how many alerts this
 	 * single alert groups */
-	unsigned int    grouped_alarms_count;
+	unsigned int    grouped_alerts_count;
 
 	/** Hyperalert information, pre-conditions
 	 * and post-conditions*/
 	AI_hyperalert_info  *hyperalert;
 
-	/* 'Parent' correlated alert in the chain,
+	/* Parent alerts in the chain, if any */
-	 * if any*/
+	struct _AI_snort_alert  **parent_alerts;
-	struct _AI_snort_alert  *previous_correlated;
+
+	/* Number of parent alerts */
+	unsigned int        n_parent_alerts;
 
 	/** Array of directly correlated 'derived'
 	 * alerts from the current one, if any */