From 997ebcbcd8b3c99a9478e1b48531ad01e62411f4 Mon Sep 17 00:00:00 2001 From: BlackLight Date: Sat, 11 Sep 2010 12:45:30 +0200 Subject: [PATCH] Sept 11 2010 commit --- TODO | 4 +- alert_parser.c | 7 + cluster.c | 14 +- correlation.c | 136 ++++++++-- db.c | 10 +- doc/html/alert__parser_8c.html | 18 +- doc/html/annotated.html | 7 +- doc/html/classes.html | 9 +- doc/html/cluster_8c.html | 3 +- doc/html/correlation_8c.html | 18 +- doc/html/db_8c.html | 2 +- doc/html/db_8h.html | 2 +- doc/html/db_8h_source.html | 2 +- doc/html/files.html | 2 +- doc/html/functions.html | 40 ++- doc/html/functions_vars.html | 40 ++- doc/html/globals.html | 38 ++- doc/html/globals_defs.html | 2 +- doc/html/globals_enum.html | 2 +- doc/html/globals_eval.html | 2 +- doc/html/globals_func.html | 23 +- doc/html/globals_type.html | 2 +- doc/html/globals_vars.html | 13 +- doc/html/group__alert__parser.html | 2 +- doc/html/group__cluster.html | 16 +- doc/html/group__correlation.html | 131 ++++++++- doc/html/group__regex.html | 94 ++++++- doc/html/group__spp__ai.html | 2 +- doc/html/group__stream.html | 2 +- doc/html/index.html | 2 +- doc/html/modules.html | 2 +- doc/html/mysql_8c.html | 2 +- doc/html/regex_8c.html | 6 +- doc/html/search/all_5f.html | 38 ++- doc/html/search/all_61.html | 120 +++++---- doc/html/search/all_62.html | 10 +- doc/html/search/all_63.html | 18 +- doc/html/search/all_67.html | 2 +- doc/html/search/all_68.html | 13 +- doc/html/search/all_6b.html | 4 +- doc/html/search/all_6c.html | 10 + doc/html/search/all_6e.html | 8 +- doc/html/search/all_70.html | 8 +- doc/html/search/all_72.html | 2 +- doc/html/search/all_73.html | 26 +- doc/html/search/classes_61.html | 21 +- doc/html/search/functions_5f.html | 30 ++- doc/html/search/functions_73.html | 36 +-- doc/html/search/search.js | 6 +- doc/html/search/variables_61.html | 23 +- doc/html/search/variables_62.html | 26 ++ doc/html/search/variables_63.html | 16 +- doc/html/search/variables_67.html | 2 +- doc/html/search/variables_68.html | 11 +- doc/html/search/variables_6b.html | 4 +- doc/html/search/variables_6c.html | 10 + doc/html/search/variables_6e.html | 8 +- doc/html/search/variables_70.html | 8 +- doc/html/search/variables_72.html | 2 +- doc/html/search/variables_73.html | 2 +- doc/html/sf__preproc__info_8h.html | 2 +- doc/html/sf__preproc__info_8h_source.html | 23 +- doc/html/spp__ai_8c.html | 2 +- doc/html/spp__ai_8h.html | 9 +- doc/html/spp__ai_8h_source.html | 292 +++++++++++---------- doc/html/stream_8c.html | 2 +- doc/html/structAI__alert__correlation.html | 148 +++++++++++ doc/html/structAI__config.html | 2 +- doc/html/structAI__hyperalert__info.html | 177 +++++++++++++ doc/html/structAI__hyperalert__key.html | 129 +++++++++ doc/html/struct__AI__snort__alert.html | 21 +- doc/html/struct__hierarchy__node.html | 4 +- doc/html/structattribute__key.html | 2 +- doc/html/structattribute__value.html | 2 +- doc/html/structpkt__info.html | 2 +- doc/html/structpkt__key.html | 2 +- doc/latex/alert__parser_8c.tex | 7 + doc/latex/annotated.tex | 5 +- doc/latex/cluster_8c.tex | 2 + doc/latex/correlation_8c.tex | 18 +- doc/latex/doxygen.sty | 4 +- doc/latex/group__cluster.tex | 7 + doc/latex/group__correlation.tex | 89 +++++-- doc/latex/group__regex.tex | 48 +++- doc/latex/refman.tex | 7 +- doc/latex/regex_8c.tex | 6 +- doc/latex/spp__ai_8h.tex | 10 +- doc/latex/structAI__alert__correlation.tex | 46 ++++ doc/latex/structAI__hyperalert__info.tex | 64 +++++ doc/latex/structAI__hyperalert__key.tex | 43 +++ doc/latex/struct__AI__snort__alert.tex | 15 +- doc/latex/struct__hierarchy__node.tex | 3 + 92 files changed, 1856 insertions(+), 456 deletions(-) create mode 100644 doc/html/search/variables_62.html create mode 100644 doc/html/structAI__alert__correlation.html create mode 100644 doc/html/structAI__hyperalert__info.html create mode 100644 doc/html/structAI__hyperalert__key.html create mode 100644 doc/latex/structAI__alert__correlation.tex create mode 100644 doc/latex/structAI__hyperalert__info.tex create mode 100644 doc/latex/structAI__hyperalert__key.tex diff --git a/TODO b/TODO index 3e22cc6..7703c4e 100644 --- a/TODO +++ b/TODO @@ -1,4 +1,6 @@ +- Correlation macros valid also for hierarchies flags +- Bayesian learning among alerts in alert log + - Managing clusters for addresses, timestamps (and more?) - Dynamic cluster_min_size algorithm -- Alerts for port scan, grouped alerts, UDP and ICMP too diff --git a/alert_parser.c b/alert_parser.c index 515533d..1a93bb4 100644 --- a/alert_parser.c +++ b/alert_parser.c @@ -29,6 +29,7 @@ PRIVATE AI_snort_alert *alerts = NULL; PRIVATE FILE *alert_fp = NULL; +PRIVATE BOOL lock_flag = false; /** \defgroup alert_parser Parse the alert log into binary structures * @{ */ @@ -105,6 +106,9 @@ AI_file_alertparser_thread ( void* arg ) inotify_rm_watch ( ifd, wd ); close ( ifd ); + /* Set the lock flag to true until it's done with alert parsing */ + lock_flag = true; + while ( !feof ( alert_fp )) { fgets ( line, sizeof(line), alert_fp ); @@ -300,6 +304,8 @@ AI_file_alertparser_thread ( void* arg ) matches = NULL; } } + + lock_flag = false; } pthread_exit ((void*) 0 ); @@ -345,6 +351,7 @@ _AI_copy_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_alerts () { + while ( lock_flag ); return _AI_copy_alerts ( alerts ); } /* ----- end of function AI_get_alerts ----- */ diff --git a/cluster.c b/cluster.c index fc7c2ea..491a024 100644 --- a/cluster.c +++ b/cluster.c @@ -45,6 +45,7 @@ typedef struct { PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL }; PRIVATE AI_config *_config = NULL; PRIVATE AI_snort_alert *alert_log = NULL; +PRIVATE BOOL lock_flag = false; /** @@ -373,11 +374,19 @@ _AI_cluster_thread ( void* arg ) /* Between an execution of the thread and the next one, sleep for alert_clustering_interval seconds */ sleep ( _config->alertClusteringInterval ); + /* Set the lock over the alert log until it's done with the clustering operation */ + lock_flag = true; + /* Free the current alert log and get the latest one */ - AI_free_alerts ( alert_log ); + if ( alert_log ) + { + AI_free_alerts ( alert_log ); + alert_log = NULL; + } if ( !( alert_log = get_alerts() )) { + lock_flag = false; continue; } @@ -481,6 +490,8 @@ _AI_cluster_thread ( void* arg ) /* break; */ } while ( old_alert_count != alert_count ); + lock_flag = false; + if ( !( cluster_fp = fopen ( _config->clusterfile, "w" )) ) { pthread_exit ((void*) 0 ); @@ -650,6 +661,7 @@ _AI_copy_clustered_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_clustered_alerts () { + while ( lock_flag ); return _AI_copy_clustered_alerts ( alert_log ); } /* ----- end of function AI_get_clustered_alerts ----- */ diff --git a/correlation.c b/correlation.c index a7ee901..be3b037 100644 --- a/correlation.c +++ b/correlation.c @@ -34,9 +34,62 @@ /** Enumeration for the types of XML tags */ enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, TAG_NUM }; -PRIVATE AI_hyperalert_info *hyperalerts = NULL; -PRIVATE AI_config *conf = NULL; -PRIVATE AI_snort_alert *alerts = NULL; +/** Struct representing the correlation between all the couples of alerts */ +typedef struct { + /** First alert */ + AI_snort_alert *a; + + /** Second alert */ + AI_snort_alert *b; + + /** Correlation coefficient */ + double correlation; + + /** Make the struct 'hashable' */ + UT_hash_handle hh; +} AI_alert_correlation; + +PRIVATE AI_hyperalert_info *hyperalerts = NULL; +PRIVATE AI_config *conf = NULL; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE AI_alert_correlation *correlation_table = NULL; +PRIVATE BOOL lock_flag = false; + +/** + * \brief Compute the correlation coefficient between two alerts, as #INTERSECTION(pre(B), post(A) / #UNION(pre(B), post(A)) + * \param a Alert a + * \param b Alert b + * \return The correlation coefficient between A and B as coefficient in [0,1] + */ + +double +_AI_correlation_coefficient ( AI_snort_alert *a, AI_snort_alert *b ) +{ + unsigned int i, j, + n_intersection = 0, + n_union = 0; + + if ( !a->hyperalert || !b->hyperalert ) + return 0.0; + + if ( a->hyperalert->n_postconds == 0 || b->hyperalert->n_preconds == 0 ) + return 0.0; + + n_union = a->hyperalert->n_postconds + b->hyperalert->n_preconds; + + for ( i=0; i < a->hyperalert->n_postconds; i++ ) + { + for ( j=0; j < b->hyperalert->n_preconds; j++ ) + { + if ( !strcasecmp ( a->hyperalert->postconds[i], b->hyperalert->preconds[j] )) + { + n_intersection += 2; + } + } + } + + return (double) ((double) n_intersection / (double) n_union ); +} /* ----- end of function _AI_correlation_coefficient ----- */ /** * \brief Substitute the macros in hyperalert pre-conditions and post-conditions with their associated values @@ -300,9 +353,13 @@ AI_alert_correlation_thread ( void *arg ) int i; struct stat st; AI_hyperalert_key key; - AI_hyperalert_info *hyp = NULL; - AI_snort_alert *tmp = NULL; - FILE *fp; + AI_hyperalert_info *hyp = NULL; + AI_snort_alert *alert_iterator = NULL, + *alert_iterator2 = NULL; + + FILE *fp = fopen ( "/home/blacklight/LOG", "w" ); + fclose ( fp ); + conf = (AI_config*) arg; while ( 1 ) @@ -317,15 +374,27 @@ AI_alert_correlation_thread ( void *arg ) return ( void* ) 0; } - if ( !( alerts = AI_get_clustered_alerts() )) - continue; + /* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */ + lock_flag = true; - for ( tmp = alerts; tmp; tmp = tmp->next ) + if ( alerts ) + { + AI_free_alerts ( alerts ); + alerts = NULL; + } + + if ( !( alerts = AI_get_clustered_alerts() )) + { + lock_flag = false; + continue; + } + + for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next ) { /* Check if my hash table of hyperalerts already contains info about this alert */ - key.gid = tmp->gid; - key.sid = tmp->sid; - key.rev = tmp->rev; + key.gid = alert_iterator->gid; + key.sid = alert_iterator->sid; + key.rev = alert_iterator->rev; HASH_FIND ( hh, hyperalerts, &key, sizeof ( AI_hyperalert_key ), hyp ); /* If not, try to read info from the XML file, if it exists */ @@ -340,34 +409,43 @@ AI_alert_correlation_thread ( void *arg ) } /* Fill the hyper alert info for the current alert */ - if ( !( tmp->hyperalert = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) + if ( !( alert_iterator->hyperalert = ( AI_hyperalert_info* ) malloc ( sizeof ( AI_hyperalert_info )))) _dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ ); - tmp->hyperalert->key = hyp->key; - tmp->hyperalert->n_preconds = hyp->n_preconds; - tmp->hyperalert->n_postconds = hyp->n_postconds; + alert_iterator->hyperalert->key = hyp->key; + alert_iterator->hyperalert->n_preconds = hyp->n_preconds; + alert_iterator->hyperalert->n_postconds = hyp->n_postconds; - if ( !( tmp->hyperalert->preconds = ( char** ) malloc ( tmp->hyperalert->n_preconds * sizeof ( char* )))) + if ( !( alert_iterator->hyperalert->preconds = ( char** ) malloc ( alert_iterator->hyperalert->n_preconds * sizeof ( char* )))) _dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ ); - for ( i=0; i < tmp->hyperalert->n_preconds; i++ ) - tmp->hyperalert->preconds[i] = strdup ( hyp->preconds[i] ); + for ( i=0; i < alert_iterator->hyperalert->n_preconds; i++ ) + alert_iterator->hyperalert->preconds[i] = strdup ( hyp->preconds[i] ); - if ( !( tmp->hyperalert->postconds = ( char** ) malloc ( tmp->hyperalert->n_postconds * sizeof ( char* )))) + if ( !( alert_iterator->hyperalert->postconds = ( char** ) malloc ( alert_iterator->hyperalert->n_postconds * sizeof ( char* )))) _dpd.fatalMsg ( "AIPreproc: Fatal memory allocation error at %s:%d\n", __FILE__, __LINE__ ); - for ( i=0; i < tmp->hyperalert->n_postconds; i++ ) - tmp->hyperalert->postconds[i] = strdup ( hyp->postconds[i] ); + for ( i=0; i < alert_iterator->hyperalert->n_postconds; i++ ) + alert_iterator->hyperalert->postconds[i] = strdup ( hyp->postconds[i] ); - _AI_macro_subst ( &tmp ); - - fp = fopen ( "/home/blacklight/LOG", "a" ); - fprintf ( fp, "pre: %s\n", (tmp->hyperalert->n_preconds > 0) ? tmp->hyperalert->preconds[0] : "()" ); - fprintf ( fp, "post: %s\n", (tmp->hyperalert->n_postconds > 0) ? tmp->hyperalert->postconds[0] : "()" ); - fclose ( fp ); + _AI_macro_subst ( &alert_iterator ); } - AI_free_alerts ( alerts ); + for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next ) + { + for ( alert_iterator2 = alerts; alert_iterator2; alert_iterator2 = alert_iterator2->next ) + { + if ( alert_iterator != alert_iterator2 ) + { + fp = fopen ( "/home/blacklight/LOG", "a" ); + fprintf ( fp, "alert1: (%s), alert2: (%s)\n", alert_iterator->desc, alert_iterator2->desc ); + fprintf ( fp, "correlation (alert1, alert2): %f\n\n", _AI_correlation_coefficient ( alert_iterator, alert_iterator2 )); + fclose ( fp ); + } + } + } + + lock_flag = false; } pthread_exit (( void* ) 0 ); diff --git a/db.c b/db.c index 0d26f46..1f84741 100644 --- a/db.c +++ b/db.c @@ -31,7 +31,8 @@ PRIVATE AI_config *config; -PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE BOOL lock_flag = false; /** pthread mutex for accessing database data */ PRIVATE pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -77,6 +78,10 @@ AI_db_alertparser_thread ( void *arg ) while ( 1 ) { sleep ( config->databaseParsingInterval ); + + /* Set the lock flag to true until it's done with alert parsing */ + lock_flag = true; + memset ( query, 0, sizeof ( query )); snprintf ( query, sizeof (query), "select cid, unix_timestamp(timestamp), signature from event where cid > %d " "and unix_timestamp(timestamp) > %ld order by cid", latest_cid, latest_time ); @@ -93,6 +98,7 @@ AI_db_alertparser_thread ( void *arg ) DB_close(); _dpd.fatalMsg ( "AIPreproc: Could not store the query result at %s:%d\n", __FILE__, __LINE__ ); } else if ( rows == 0 ) { + lock_flag = false; continue; } @@ -218,6 +224,7 @@ AI_db_alertparser_thread ( void *arg ) } } + lock_flag = false; DB_free_result ( res ); latest_time = time ( NULL ); } @@ -265,6 +272,7 @@ _AI_db_copy_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_db_get_alerts () { + while ( lock_flag ); return _AI_db_copy_alerts ( alerts ); } /* ----- end of function AI_db_get_alerts ----- */ diff --git a/doc/html/alert__parser_8c.html b/doc/html/alert__parser_8c.html index 58efaf9..a14b5c1 100644 --- a/doc/html/alert__parser_8c.html +++ b/doc/html/alert__parser_8c.html @@ -75,6 +75,7 @@ Functions Variables PRIVATE AI_snort_alertalerts = NULL PRIVATE FILE * alert_fp = NULL +PRIVATE BOOL lock_flag = false

Variable Documentation

@@ -95,7 +96,20 @@ Variables
- + + +
PRIVATE AI_snort_alert* alerts = NULLPRIVATE AI_snort_alert* alerts = NULL
+
+
+ +
+ + +
+
+ + +
PRIVATE BOOL lock_flag = false
@@ -118,7 +132,7 @@ Variables
-