diff --git a/README b/README index e84926a..be801e5 100644 --- a/README +++ b/README @@ -147,7 +147,8 @@ following: preprocessor ai: \ hashtable_cleanup_interval 300 \ tcp_stream_expire_interval 300 \ - alertfile "/home/youruser/local/snort/log/alert" \ + alertfile "/your/snort/dir/log/alert" \ + alert_history_file "/your/snort/dir/log/alert_history" \ alert_clustering_interval 300 \ correlation_graph_interval 300 \ correlation_rules_dir "/your/snort/dir/etc/corr_rules" \ @@ -178,6 +179,10 @@ stream as "expired", if no more packets are received inside of that and it's not - alertfile: The file where Snort saves its alerts, if they are saved to a file and not to a database (default if not specified: /var/log/snort/alert) +- alert_history_file: The file keeping track of the history, in binary format, +of all the alerts received by the IDS, so that the module can build some +statistical correlation inferences over the past + - alert_clustering_interval: The interval that should occur from the clustering of the alerts in the log according to the provided clustering hierarchies and the next one (default if not specified: 300 seconds) diff --git a/TODO b/TODO index 46302e7..c247324 100644 --- a/TODO +++ b/TODO @@ -2,7 +2,6 @@ AVERAGE/HIGH PRIORITY: ====================== -- Dynamic k parameter in correlation threshold - Testing more scenarios, making more hyperalert models - Bayesian learning among alerts in alert log - libgc support diff --git a/alert_history.c b/alert_history.c new file mode 100644 index 0000000..d19b944 --- /dev/null +++ b/alert_history.c @@ -0,0 +1,21 @@ +/* + * ===================================================================================== + * + * Filename: alert_history.c + * + * Description: Manages the history of alerts on a binary file + * + * Version: 0.1 + * Created: 18/09/2010 21:02:15 + * Revision: none + * Compiler: gcc + * + * Author: BlackLight (http://0x00.ath.cx), + * Licence: GNU GPL v.3 + * Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE! + * + * ===================================================================================== + */ + +#include "spp_ai.h" + diff --git a/alert_parser.c b/alert_parser.c index da72ed0..87e416e 100644 --- a/alert_parser.c +++ b/alert_parser.c @@ -29,9 +29,9 @@ #include -PRIVATE AI_snort_alert *alerts = NULL; -PRIVATE FILE *alert_fp = NULL; -PRIVATE BOOL lock_flag = false; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE FILE *alert_fp = NULL; +PRIVATE pthread_mutex_t mutex; /** \defgroup alert_parser Parse the alert log into binary structures * @{ */ @@ -78,6 +78,8 @@ AI_file_alertparser_thread ( void* arg ) AI_snort_alert *tmp = NULL; BOOL in_alert = false; + pthread_mutex_init ( &mutex, NULL ); + while ( 1 ) { #ifndef MACOS @@ -132,6 +134,7 @@ AI_file_alertparser_thread ( void* arg ) fd = fileno(alert_fp); } } + /* * Cause the thread to wait until a new file modification (a new alert). */ @@ -139,6 +142,7 @@ AI_file_alertparser_thread ( void* arg ) usleep(100); fstats( fd, &stats ); } + /* * The first time the thread is called, the flow exits instantly from the while, * so this first time the stats structure has to be initialized properly. @@ -153,7 +157,6 @@ AI_file_alertparser_thread ( void* arg ) #endif /* Set the lock flag to true until it's done with alert parsing */ - lock_flag = true; while ( !feof ( alert_fp )) { @@ -205,6 +208,8 @@ AI_file_alertparser_thread ( void* arg ) if ( !in_alert ) { + pthread_mutex_lock ( &mutex ); + if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 ) { in_alert = true; @@ -351,9 +356,11 @@ AI_file_alertparser_thread ( void* arg ) } } - lock_flag = false; + pthread_mutex_unlock ( &mutex ); + /* AI_alert_serialize ( alert, conf ); */ } + pthread_mutex_destroy ( &mutex ); pthread_exit ((void*) 0 ); return (void*) 0; } /* ----- end of function AI_file_alertparser_thread ----- */ @@ -397,8 +404,13 @@ _AI_copy_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_alerts () { - while ( lock_flag ); - return _AI_copy_alerts ( alerts ); + AI_snort_alert *alerts_copy; + + pthread_mutex_lock ( &mutex ); + alerts_copy = _AI_copy_alerts ( alerts ); + pthread_mutex_unlock ( &mutex ); + + return alerts_copy; } /* ----- end of function AI_get_alerts ----- */ diff --git a/cluster.c b/cluster.c index 5b85f96..2f094ed 100644 --- a/cluster.c +++ b/cluster.c @@ -50,11 +50,10 @@ typedef struct { } AI_alert_occurrence; -PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL }; -PRIVATE AI_config *_config = NULL; -PRIVATE AI_snort_alert *alert_log = NULL; -PRIVATE BOOL lock_flag = false; - +PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL }; +PRIVATE AI_config *_config = NULL; +PRIVATE AI_snort_alert *alert_log = NULL; +PRIVATE pthread_mutex_t mutex; /** * \brief Function that picks up the heuristic value for a clustering attribute in according to Julisch's heuristic (ACM, Vol.2, No.3, 09 2002, pag.124) @@ -373,7 +372,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp ) { AI_snort_alert *tmp; char ip[INET_ADDRSTRLEN]; - char *timestamp; + char timestamp[128]; + struct tm *_tm; for ( tmp = log; tmp; tmp = tmp->next ) { @@ -384,8 +384,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp ) fprintf ( fp, "[Priority: %d]\n", tmp->priority ); - timestamp = ctime ( &tmp->timestamp ); - timestamp[ strlen(timestamp)-1 ] = 0; + _tm = localtime ( &tmp->timestamp ); + strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm ); fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp ); if ( h_root[src_addr] && tmp->h_node[src_addr] ) @@ -445,13 +445,15 @@ _AI_cluster_thread ( void* arg ) int single_alerts_count = 0; double heterogeneity = 0; + pthread_mutex_init ( &mutex, NULL ); + while ( 1 ) { /* Between an execution of the thread and the next one, sleep for alert_clustering_interval seconds */ sleep ( _config->alertClusteringInterval ); /* Set the lock over the alert log until it's done with the clustering operation */ - lock_flag = true; + pthread_mutex_lock ( &mutex ); /* Free the current alert log and get the latest one */ if ( alert_log ) @@ -462,7 +464,7 @@ _AI_cluster_thread ( void* arg ) if ( !( alert_log = get_alerts() )) { - lock_flag = false; + pthread_mutex_unlock ( &mutex ); continue; } @@ -561,7 +563,7 @@ _AI_cluster_thread ( void* arg ) alert_count -= _AI_merge_alerts ( &alert_log ); } while ( old_alert_count != alert_count ); - lock_flag = false; + pthread_mutex_unlock ( &mutex ); if ( !( cluster_fp = fopen ( _config->clusterfile, "w" )) ) { @@ -732,8 +734,13 @@ _AI_copy_clustered_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_get_clustered_alerts () { - for ( ; lock_flag; usleep(100) ); - return _AI_copy_clustered_alerts ( alert_log ); + AI_snort_alert *alerts_copy; + + pthread_mutex_lock ( &mutex ); + alerts_copy = _AI_copy_clustered_alerts ( alert_log ); + pthread_mutex_unlock ( &mutex ); + + return alerts_copy; } /* ----- end of function AI_get_clustered_alerts ----- */ /** @} */ diff --git a/correlation.c b/correlation.c index 278de0c..bc784ea 100644 --- a/correlation.c +++ b/correlation.c @@ -70,7 +70,7 @@ PRIVATE AI_hyperalert_info *hyperalerts = NULL; PRIVATE AI_config *conf = NULL; PRIVATE AI_snort_alert *alerts = NULL; PRIVATE AI_alert_correlation *correlation_table = NULL; -PRIVATE BOOL lock_flag = false; +PRIVATE pthread_mutex_t mutex; /** * \brief Clean up the correlation hash table @@ -713,6 +713,7 @@ AI_alert_correlation_thread ( void *arg ) #endif conf = (AI_config*) arg; + pthread_mutex_init ( &mutex, NULL ); while ( 1 ) { @@ -727,7 +728,7 @@ AI_alert_correlation_thread ( void *arg ) } /* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */ - lock_flag = true; + pthread_mutex_lock ( &mutex ); if ( alerts ) { @@ -737,7 +738,7 @@ AI_alert_correlation_thread ( void *arg ) if ( !( alerts = AI_get_clustered_alerts() )) { - lock_flag = false; + pthread_mutex_unlock ( &mutex ); continue; } @@ -894,7 +895,7 @@ AI_alert_correlation_thread ( void *arg ) #endif } - lock_flag = false; + pthread_mutex_unlock ( &mutex ); } pthread_exit (( void* ) 0 ); diff --git a/db.c b/db.c index f2929f7..8868305 100644 --- a/db.c +++ b/db.c @@ -30,12 +30,9 @@ * @{ */ -PRIVATE AI_config *config; -PRIVATE AI_snort_alert *alerts = NULL; -PRIVATE BOOL lock_flag = false; - -/** pthread mutex for accessing database data */ -PRIVATE pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER; +PRIVATE AI_config *config; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE pthread_mutex_t mutex; /** * \brief Thread for parsing alerts from a database @@ -65,7 +62,7 @@ AI_db_alertparser_thread ( void *arg ) } config = ( AI_config* ) arg; - pthread_mutex_lock ( &db_mutex ); + pthread_mutex_init ( &mutex, NULL ); if ( !DB_init ( config )) { @@ -73,14 +70,10 @@ AI_db_alertparser_thread ( void *arg ) config->dbname, config->dbhost ); } - pthread_mutex_unlock ( &db_mutex ); - while ( 1 ) { sleep ( config->databaseParsingInterval ); - - /* Set the lock flag to true until it's done with alert parsing */ - lock_flag = true; + pthread_mutex_lock ( &mutex ); memset ( query, 0, sizeof ( query )); snprintf ( query, sizeof (query), "select cid, unix_timestamp(timestamp), signature from event where cid > %d " @@ -98,7 +91,7 @@ AI_db_alertparser_thread ( void *arg ) DB_close(); _dpd.fatalMsg ( "AIPreproc: Could not store the query result at %s:%d\n", __FILE__, __LINE__ ); } else if ( rows == 0 ) { - lock_flag = false; + pthread_mutex_unlock ( &mutex ); continue; } @@ -224,7 +217,7 @@ AI_db_alertparser_thread ( void *arg ) } } - lock_flag = false; + pthread_mutex_unlock ( &mutex ); DB_free_result ( res ); latest_time = time ( NULL ); } @@ -272,8 +265,13 @@ _AI_db_copy_alerts ( AI_snort_alert *node ) AI_snort_alert* AI_db_get_alerts () { - while ( lock_flag ); - return _AI_db_copy_alerts ( alerts ); + AI_snort_alert *alerts_copy; + + pthread_mutex_lock ( &mutex ); + alerts_copy = _AI_db_copy_alerts ( alerts ); + pthread_mutex_unlock ( &mutex ); + + return alerts_copy; } /* ----- end of function AI_db_get_alerts ----- */ /** @} */ diff --git a/spp_ai.c b/spp_ai.c index a6fc704..2120f95 100644 --- a/spp_ai.c +++ b/spp_ai.c @@ -135,10 +135,11 @@ static AI_config * AI_parse(char *args) { char *arg; char *match; - char alertfile[1024] = { 0 }; - char clusterfile[1024] = { 0 }; - char corr_rules_dir[1024] = { 0 }; - char corr_alerts_dir[1024] = { 0 }; + char alertfile[1024] = { 0 }; + char clusterfile[1024] = { 0 }; + char corr_rules_dir[1024] = { 0 }; + char corr_alerts_dir[1024] = { 0 }; + char alert_history_file[1024] = { 0 }; char **matches = NULL; int nmatches = 0; @@ -160,6 +161,7 @@ static AI_config * AI_parse(char *args) unsigned long cleanup_interval = 0, stream_expire_interval = 0, alertfile_len = 0, + alert_history_file_len = 0, clusterfile_len = 0, corr_rules_dir_len = 0, corr_alerts_dir_len = 0, @@ -176,7 +178,8 @@ static AI_config * AI_parse(char *args) has_clusterfile = false, has_corr_rules_dir = false, has_clustering = false, - has_database_log = false; + has_database_log = false, + has_alert_history_file = false; AI_config *config = NULL; @@ -336,6 +339,38 @@ static AI_config * AI_parse(char *args) } } + /* Parsing the alert_history_file option */ + if (( arg = (char*) strcasestr( args, "alert_history_file" ) )) + { + for ( arg += strlen("alert_history_file"); + *arg && *arg != '"'; + arg++ ); + + if ( !(*(arg++)) ) + { + _dpd.fatalMsg("AIPreproc: alert_history_file option used but no filename specified\n"); + } + + for ( alert_history_file[ (++alert_history_file_len)-1 ] = *arg; + *arg && *arg != '"' && alert_history_file_len < 1024; + arg++, alert_history_file[ (++alert_history_file_len)-1 ] = *arg ); + + if ( alert_history_file[0] == 0 || alert_history_file_len <= 1 ) { + has_alert_history_file = false; + } else { + if ( alert_history_file_len >= 1024 ) { + _dpd.fatalMsg("AIPreproc: alert_history_file path too long ( >= 1024 )\n"); + } else if ( strlen( alert_history_file ) == 0 ) { + has_alert_history_file = false; + } else { + has_alert_history_file = true; + alert_history_file [ alert_history_file_len-1 ] = 0; + strncpy ( config->alert_history_file, alert_history_file, alert_history_file_len ); + _dpd.logMsg(" alert_history_file path: %s\n", config->alert_history_file); + } + } + } + /* Parsing the clusterfile option */ if (( arg = (char*) strcasestr( args, "clusterfile" ) )) { @@ -767,6 +802,12 @@ static AI_config * AI_parse(char *args) alertparser_thread = AI_file_alertparser_thread; } + if ( !has_alert_history_file ) + { + strncpy ( config->alert_history_file, DEFAULT_ALERT_HISTORY_FILE, sizeof ( config->alert_history_file )); + has_alert_history_file = true; + } + if ( has_clustering ) { if ( ! hierarchy_nodes ) diff --git a/spp_ai.h b/spp_ai.h index 98f43e2..e097b5e 100644 --- a/spp_ai.h +++ b/spp_ai.h @@ -57,6 +57,9 @@ /** Default directory for placing correlated alerts information (.dot and possibly .png files) */ #define DEFAULT_CORR_ALERTS_DIR "/var/log/snort/correlated_alerts" +/** Default path to alert history binary file, used for bayesian statistical correlation over alerts */ +#define DEFAULT_ALERT_HISTORY_FILE "/var/log/snort/alert_history" + /** Default correlation threshold coefficient for correlating two hyperalerts */ #define DEFAULT_CORR_THRESHOLD 0.5 @@ -144,6 +147,9 @@ typedef struct /** Alert file */ char alertfile[1024]; + /** Alert history binary file */ + char alert_history_file[1024]; + /** Clustered alerts file */ char clusterfile[1024]; @@ -305,6 +311,9 @@ struct pkt_info* AI_get_stream_by_key ( struct pkt_key ); AI_snort_alert* AI_get_alerts ( void ); AI_snort_alert* AI_get_clustered_alerts ( void ); +void AI_serialize_alert ( AI_snort_alert*, AI_config* ); +void AI_deserialize_alert ( AI_snort_alert*, AI_config* ); + /** Function pointer to the function used for getting the alert list (from log file, db, ...) */ extern AI_snort_alert* (*get_alerts)(void);