Grouped alert info inside the cluster, better graph management in hyperalerts

This commit is contained in:
BlackLight 2010-09-16 23:21:38 +02:00
parent b40dca6b14
commit 5b471d9003
5 changed files with 70 additions and 21 deletions

24
TODO
View file

@ -1,9 +1,25 @@
======================
AVERAGE/HIGH PRIORITY:
======================
- Dynamic cluster_min_size algorithm
- Dynamic k parameter in correlation threshold
- Testing more scenarios, making more hyperalert models
- Bayesian learning among alerts in alert log - Bayesian learning among alerts in alert log
- libgc support - libgc support
+ PostgreSQL support [DONE]
- Dynamic k parameter in correlation threshold =============
- Dynamic cluster_min_size algorithm LOW PRIORITY:
+ Regex comp cache [DONE] =============
- Managing clusters for addresses, timestamps (and more?) - Managing clusters for addresses, timestamps (and more?)
=====
DONE:
=====
+ PostgreSQL support
+ Regex comp cache
+ Managing hyperalert graph connection inside the alert structure itself
+ Keeping track of all the streams and alerts even after clustered

View file

@ -417,6 +417,20 @@ AI_free_alerts ( AI_snort_alert *node )
if ( node->next ) if ( node->next )
AI_free_alerts ( node->next ); AI_free_alerts ( node->next );
/* if ( node->grouped_alerts ) */
/* { */
/* for ( i=0; i < node->grouped_alerts_count; i++ ) */
/* { */
/* if ( node->grouped_alerts[i] ) */
/* { */
/* free ( node->grouped_alerts[i] ); */
/* node->grouped_alerts[i] = NULL; */
/* } */
/* } */
/* free ( node->grouped_alerts ); */
/* } */
if ( node->hyperalert ) if ( node->hyperalert )
{ {
for ( i=0; i < node->hyperalert->n_preconds; i++ ) for ( i=0; i < node->hyperalert->n_preconds; i++ )
@ -431,6 +445,9 @@ AI_free_alerts ( AI_snort_alert *node )
node->hyperalert = NULL; node->hyperalert = NULL;
} }
if ( node->parent_alerts )
free ( node->parent_alerts );
if ( node->derived_alerts ) if ( node->derived_alerts )
free ( node->derived_alerts ); free ( node->derived_alerts );

View file

@ -268,12 +268,15 @@ _AI_merge_alerts ( AI_snort_alert **log )
{ {
if ( _AI_equal_alarms ( tmp, tmp2->next )) if ( _AI_equal_alarms ( tmp, tmp2->next ))
{ {
tmp3 = tmp2->next->next; if ( !( tmp->grouped_alerts = ( AI_snort_alert** ) realloc ( tmp->grouped_alerts, (++(tmp->grouped_alerts_count)) * sizeof ( AI_snort_alert* ))))
free ( tmp2->next ); _dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
tmp2->next = tmp3;
tmp->grouped_alarms_count++; tmp->grouped_alerts[ tmp->grouped_alerts_count - 1 ] = tmp2->next;
count++; count++;
tmp3 = tmp2->next->next;
/* free ( tmp2->next ); */
tmp2->next = tmp3;
} }
} }
@ -311,7 +314,7 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
timestamp = ctime ( &tmp->timestamp ); timestamp = ctime ( &tmp->timestamp );
timestamp[ strlen(timestamp)-1 ] = 0; timestamp[ strlen(timestamp)-1 ] = 0;
fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alarms_count, timestamp ); fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
if ( h_root[src_addr] && tmp->h_node[src_addr] ) if ( h_root[src_addr] && tmp->h_node[src_addr] )
{ {
@ -395,13 +398,13 @@ _AI_cluster_thread ( void* arg )
for ( tmp = alert_log, alert_count=0; tmp; tmp = tmp->next, alert_count++ ) for ( tmp = alert_log, alert_count=0; tmp; tmp = tmp->next, alert_count++ )
{ {
/* If an alert has an unitialized "grouped alarms count", set its counter to 1 (it only groupes the current alert) */ /* If an alert has an unitialized "grouped alarms count", set its counter to 1 (it only groupes the current alert) */
if ( tmp->grouped_alarms_count == 0 ) if ( tmp->grouped_alerts_count == 0 )
{ {
tmp->grouped_alarms_count = 1; tmp->grouped_alerts_count = 1;
} }
/* If the current alarm already group at least min_size alarms, then no need to do further clusterization */ /* If the current alarm already group at least min_size alarms, then no need to do further clusterization */
if ( tmp->grouped_alarms_count >= cluster_min_size ) if ( tmp->grouped_alerts_count >= cluster_min_size )
{ {
has_small_clusters = false; has_small_clusters = false;
} }
@ -475,7 +478,7 @@ _AI_cluster_thread ( void* arg )
/* For all the alerts, the corresponing clustering value is the parent of the current one in the hierarchy */ /* For all the alerts, the corresponing clustering value is the parent of the current one in the hierarchy */
for ( tmp = alert_log; tmp; tmp = tmp->next ) for ( tmp = alert_log; tmp; tmp = tmp->next )
{ {
if ( tmp->grouped_alarms_count < cluster_min_size && tmp->h_node[best_type] ) if ( tmp->grouped_alerts_count < cluster_min_size && tmp->h_node[best_type] )
{ {
if ( tmp->h_node[best_type]->parent ) if ( tmp->h_node[best_type]->parent )
{ {

View file

@ -147,12 +147,12 @@ _AI_print_correlated_alerts ( AI_alert_correlation *corr, FILE *fp, BOOL strong
corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc, corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc,
src_addr1, src_port1, dst_addr1, dst_port1, src_addr1, src_port1, dst_addr1, dst_port1,
timestamp1, timestamp1,
corr->key.a->grouped_alarms_count, corr->key.a->grouped_alerts_count,
corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc, corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc,
src_addr2, src_port2, dst_addr2, dst_port2, src_addr2, src_port2, dst_addr2, dst_port2,
timestamp2, timestamp2,
corr->key.b->grouped_alarms_count, corr->key.b->grouped_alerts_count,
strong ? "" : "[style=dotted]" strong ? "" : "[style=dotted]"
); );
} /* ----- end of function _AI_correlation_flow_to_file ----- */ } /* ----- end of function _AI_correlation_flow_to_file ----- */
@ -687,7 +687,8 @@ AI_alert_correlation_thread ( void *arg )
{ {
int i; int i;
struct stat st; struct stat st;
char corr_dot_file[4096] = { 0 }; char corr_dot_file[4096] = { 0 },
corr_ps_file [4096] = { 0 };
double avg_correlation = 0.0, double avg_correlation = 0.0,
std_deviation = 0.0, std_deviation = 0.0,
@ -858,8 +859,11 @@ AI_alert_correlation_thread ( void *arg )
if ( !( corr->key.a->derived_alerts = ( AI_snort_alert** ) realloc ( corr->key.a->derived_alerts, (++corr->key.a->n_derived_alerts) * sizeof ( AI_snort_alert* )))) if ( !( corr->key.a->derived_alerts = ( AI_snort_alert** ) realloc ( corr->key.a->derived_alerts, (++corr->key.a->n_derived_alerts) * sizeof ( AI_snort_alert* ))))
_dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ ); _dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ );
if ( !( corr->key.b->parent_alerts = ( AI_snort_alert** ) realloc ( corr->key.b->parent_alerts, (++corr->key.b->n_parent_alerts) * sizeof ( AI_snort_alert* ))))
_dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ );
corr->key.a->derived_alerts[ corr->key.a->n_derived_alerts - 1 ] = corr->key.b; corr->key.a->derived_alerts[ corr->key.a->n_derived_alerts - 1 ] = corr->key.b;
corr->key.b->previous_correlated = corr->key.a; corr->key.b->parent_alerts [ corr->key.b->n_parent_alerts - 1 ] = corr->key.a;
_AI_print_correlated_alerts ( corr, fp, ( corr->correlation >= corr_strong_threshold )); _AI_print_correlated_alerts ( corr, fp, ( corr->correlation >= corr_strong_threshold ));
} }
} }
@ -869,6 +873,7 @@ AI_alert_correlation_thread ( void *arg )
#ifdef HAVE_LIBGVC #ifdef HAVE_LIBGVC
snprintf ( corr_png_file, sizeof ( corr_png_file ), "%s/correlated_alerts.png", conf->corr_alerts_dir ); snprintf ( corr_png_file, sizeof ( corr_png_file ), "%s/correlated_alerts.png", conf->corr_alerts_dir );
snprintf ( corr_ps_file , sizeof ( corr_ps_file ), "%s/correlated_alerts.ps" , conf->corr_alerts_dir );
if ( !( gvc = gvContext() )) if ( !( gvc = gvContext() ))
continue; continue;
@ -881,6 +886,7 @@ AI_alert_correlation_thread ( void *arg )
gvLayout ( gvc, g, "dot" ); gvLayout ( gvc, g, "dot" );
gvRenderFilename ( gvc, g, "png", corr_png_file ); gvRenderFilename ( gvc, g, "png", corr_png_file );
gvRenderFilename ( gvc, g, "ps" , corr_ps_file );
gvFreeLayout ( gvc, g ); gvFreeLayout ( gvc, g );
agclose ( g ); agclose ( g );

View file

@ -253,18 +253,25 @@ typedef struct _AI_snort_alert {
* if the clustering algorithm is used */ * if the clustering algorithm is used */
hierarchy_node *h_node[CLUSTER_TYPES]; hierarchy_node *h_node[CLUSTER_TYPES];
/** If the clustering algorithm is used,
* keep tracked of the pointers to the
* single grouped alerts */
struct _AI_snort_alert **grouped_alerts;
/** If the clustering algorithm is used, /** If the clustering algorithm is used,
* we also count how many alerts this * we also count how many alerts this
* single alert groups */ * single alert groups */
unsigned int grouped_alarms_count; unsigned int grouped_alerts_count;
/** Hyperalert information, pre-conditions /** Hyperalert information, pre-conditions
* and post-conditions*/ * and post-conditions*/
AI_hyperalert_info *hyperalert; AI_hyperalert_info *hyperalert;
/* 'Parent' correlated alert in the chain, /* Parent alerts in the chain, if any */
* if any*/ struct _AI_snort_alert **parent_alerts;
struct _AI_snort_alert *previous_correlated;
/* Number of parent alerts */
unsigned int n_parent_alerts;
/** Array of directly correlated 'derived' /** Array of directly correlated 'derived'
* alerts from the current one, if any */ * alerts from the current one, if any */