From 684f387a6ec966fdea30dfe032b6999e0dc09c7d Mon Sep 17 00:00:00 2001 From: BlackLight Date: Tue, 21 Sep 2010 16:27:46 +0200 Subject: [PATCH] Supporting alert history serialization --- Makefile.am | 1 + Makefile.in | 7 +- TODO | 1 + alert_history.c | 220 ++++++++++++++++++++++++++++++++++++++++++++++++ alert_parser.c | 136 +++++++++++++++++++++++++----- spp_ai.c | 70 ++++++++++++--- spp_ai.h | 22 +++-- 7 files changed, 421 insertions(+), 36 deletions(-) diff --git a/Makefile.am b/Makefile.am index c25d654..80f7725 100644 --- a/Makefile.am +++ b/Makefile.am @@ -16,6 +16,7 @@ include/sf_dynamic_preproc_lib.c \ include/sfPolicyUserData.c libsf_ai_preproc_la_SOURCES = \ +alert_history.c \ alert_parser.c \ cluster.c \ correlation.c \ diff --git a/Makefile.in b/Makefile.in index 0776456..2641269 100644 --- a/Makefile.in +++ b/Makefile.in @@ -75,7 +75,8 @@ am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(corr_rulesdir)" \ "$(DESTDIR)$(sharedir)" LTLIBRARIES = $(lib_LTLIBRARIES) libsf_ai_preproc_la_LIBADD = -am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_parser.lo \ +am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_history.lo \ + libsf_ai_preproc_la-alert_parser.lo \ libsf_ai_preproc_la-cluster.lo \ libsf_ai_preproc_la-correlation.lo libsf_ai_preproc_la-db.lo \ libsf_ai_preproc_la-mysql.lo libsf_ai_preproc_la-postgresql.lo \ @@ -250,6 +251,7 @@ include/sf_dynamic_preproc_lib.c \ include/sfPolicyUserData.c libsf_ai_preproc_la_SOURCES = \ +alert_history.c \ alert_parser.c \ cluster.c \ correlation.c \ @@ -371,6 +373,9 @@ distclean-compile: .c.lo: $(LTCOMPILE) -c -o $@ $< +libsf_ai_preproc_la-alert_history.lo: alert_history.c + $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-alert_history.lo `test -f 'alert_history.c' || echo '$(srcdir)/'`alert_history.c + libsf_ai_preproc_la-alert_parser.lo: alert_parser.c $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-alert_parser.lo `test -f 'alert_parser.c' || echo '$(srcdir)/'`alert_parser.c diff --git a/TODO b/TODO index c247324..55cdf98 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,7 @@ AVERAGE/HIGH PRIORITY: ====================== +- Add alerts' history serialization to db.c as well - Testing more scenarios, making more hyperalert models - Bayesian learning among alerts in alert log - libgc support diff --git a/alert_history.c b/alert_history.c index d19b944..4c649ac 100644 --- a/alert_history.c +++ b/alert_history.c @@ -19,3 +19,223 @@ #include "spp_ai.h" +#include + +typedef struct { + int gid; + int sid; + int rev; +} AI_alert_event_key; + +typedef struct _AI_alert_event { + AI_alert_event_key key; + unsigned int count; + time_t timestamp; + struct _AI_alert_event *next; + UT_hash_handle hh; +} AI_alert_event; + + +PRIVATE AI_alert_event *alerts_hash = NULL; + + +/** + * FUNCTION: AI_alerts_hash_free + * \brief Free a hash table of alert events + * \param events Hash table to be freed + */ + +void +AI_alerts_hash_free ( AI_alert_event **events ) +{ + AI_alert_event *hash_iterator = NULL, + *list_iterator = NULL, + *tmp = NULL; + + while ( *events ) + { + hash_iterator = *events; + HASH_DEL ( *events, hash_iterator ); + list_iterator = hash_iterator; + + while ( list_iterator ) + { + tmp = list_iterator->next; + free ( list_iterator ); + list_iterator = tmp; + } + + free ( hash_iterator ); + } + + *events = NULL; +} /* ----- end of function AI_alerts_hash_free ----- */ + +/** + * \brief Deserialize a alerts' hash table from the binary history file + * \param conf Configuration of the module + * \return A void* pointer (to be casted to AI_alert_event*) to the stored hash table + */ + +void* +AI_deserialize_alerts ( AI_config *conf ) +{ + FILE *fp = NULL; + struct stat st; + unsigned int i, j, + lists_count = 0, + items_count = 0; + AI_alert_event *event_iterator = NULL, + *event_prev = NULL, + *event_list = NULL; + AI_alert_event_key key; + + if ( stat ( conf->alert_history_file, &st ) < 0 ) + return NULL; + + if ( ! S_ISREG ( st.st_mode )) + _dpd.fatalMsg ( "AIPreproc: '%s' is not a regular file\n", conf->alert_history_file ); + + if ( !( fp = fopen ( conf->alert_history_file, "r" ))) + _dpd.fatalMsg ( "AIPreproc: Unable to read from the file '%s'\n", conf->alert_history_file ); + + AI_alerts_hash_free ( &alerts_hash ); + + if ( fread ( &lists_count, sizeof ( unsigned int ), 1, fp ) <= 0 ) + _dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file ); + + /* Fill the hash table reading from the file */ + for ( i=0; i < lists_count; i++ ) + { + event_iterator = NULL; + event_prev = NULL; + + if ( fread ( &items_count, sizeof ( unsigned int ), 1, fp ) <= 0 ) + _dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file ); + + for ( j=0; j < items_count; j++ ) + { + if ( j == 0 ) + { + if ( !( event_list = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event )))) + _dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ ); + + memset ( event_list, 0, sizeof ( AI_alert_event )); + event_iterator = event_list; + } else { + if ( !( event_iterator = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event )))) + _dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ ); + memset ( event_iterator, 0, sizeof ( AI_alert_event )); + } + + event_iterator->count = items_count; + + if ( fread ( &( event_iterator->key ), sizeof ( event_iterator->key ), 1, fp ) <= 0 ) + _dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file ); + + if ( fread ( &( event_iterator->timestamp ), sizeof ( event_iterator->timestamp ), 1, fp ) <= 0 ) + _dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file ); + + if ( event_prev ) + { + event_prev->next = event_iterator; + } + + event_prev = event_iterator; + } + + key = event_iterator->key; + HASH_ADD ( hh, alerts_hash, key, sizeof ( key ), event_list ); + } + + fclose ( fp ); + return (void*) alerts_hash; +} /* ----- end of function AI_deserialize_alerts ----- */ + + +/** + * \brief Serialize a buffer of alerts to the binary history file + * \param alerts_pool Buffer of alerts to be serialized + * \param alerts_pool_count Number of alerts in the buffer + * \param conf Configuration of the module + */ + +void +AI_serialize_alerts ( AI_snort_alert **alerts_pool, unsigned int alerts_pool_count, AI_config *conf ) +{ + unsigned int i, + hash_count = 0, + list_count = 0; + FILE *fp = NULL; + AI_alert_event_key key; + AI_alert_event *found = NULL, + *event = NULL, + *event_next = NULL, + *event_iterator = NULL; + + if ( !alerts_hash ) + { + AI_deserialize_alerts ( conf ); + } + + for ( i=0; i < alerts_pool_count; i++ ) + { + if ( !( event = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event )))) + _dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ ); + + memset ( event, 0, sizeof ( AI_alert_event )); + key.gid = alerts_pool[i]->gid; + key.sid = alerts_pool[i]->sid; + key.rev = alerts_pool[i]->rev; + event->key = key; + event->timestamp = alerts_pool[i]->timestamp; + + HASH_FIND ( hh, alerts_hash, &key, sizeof ( key ), found ); + + if ( !found ) + { + event->count = 1; + event->next = NULL; + HASH_ADD ( hh, alerts_hash, key, sizeof ( key ), event ); + } else { + found->count++; + event_next = NULL; + + for ( event_iterator = found; event_iterator->next; event_iterator = event_iterator->next ) + { + /* Insert the new event in cronological order */ + if ( event_iterator->next->timestamp > event->timestamp ) + { + event_next = event_iterator->next; + break; + } + } + + if ( event_iterator ) + event_iterator->next = event; + + event->next = event_next; + } + } + + hash_count = HASH_COUNT ( alerts_hash ); + + if ( !( fp = fopen ( conf->alert_history_file, "w" ))) + _dpd.fatalMsg ( "AIPreproc: Unable to write on '%s'\n", conf->alert_history_file ); + fwrite ( &hash_count, sizeof ( hash_count ), 1, fp ); + + for ( event = alerts_hash; event; event = ( AI_alert_event* ) event->hh.next ) + { + list_count = event->count; + fwrite ( &list_count, sizeof ( list_count ), 1, fp ); + + for ( event_iterator = event; event_iterator; event_iterator = event_iterator->next ) + { + fwrite ( &(event_iterator->key), sizeof ( event_iterator->key ), 1, fp ); + fwrite ( &(event_iterator->timestamp), sizeof ( event_iterator->timestamp ), 1, fp ); + } + } + + fclose ( fp ); +} /* ----- end of function AI_serialize_alerts ----- */ + diff --git a/alert_parser.c b/alert_parser.c index 87e416e..19f2913 100644 --- a/alert_parser.c +++ b/alert_parser.c @@ -29,13 +29,91 @@ #include -PRIVATE AI_snort_alert *alerts = NULL; -PRIVATE FILE *alert_fp = NULL; -PRIVATE pthread_mutex_t mutex; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE AI_snort_alert **alerts_pool = NULL; +PRIVATE AI_config *conf = NULL; +PRIVATE FILE *alert_fp = NULL; +PRIVATE unsigned int alerts_pool_count = 0; +PRIVATE pthread_mutex_t alert_mutex; +PRIVATE pthread_mutex_t alerts_pool_mutex; + /** \defgroup alert_parser Parse the alert log into binary structures * @{ */ + +/** + * \brief Serialize the pool of alerts in a separated thread + * \param arg void* pointer to the alert to be added to the pool, if any + */ + +PRIVATE void* +_AI_serializer_thread ( void *arg ) +{ + unsigned int i = 0; + AI_snort_alert *alert = NULL; + + if ( arg ) + { + alert = ( AI_snort_alert* ) arg; + + pthread_mutex_lock ( &alerts_pool_mutex ); + alerts_pool [ alerts_pool_count++ ] = alert; + pthread_mutex_unlock ( &alerts_pool_mutex ); + } + + if ( !arg || ( arg && alerts_pool_count >= conf->alert_bufsize )) + { + pthread_mutex_lock ( &alerts_pool_mutex ); + _dpd.logMsg ( "**** LOCKED ****\n" ); + AI_serialize_alerts ( alerts_pool, alerts_pool_count, conf ); + + for ( i=0; i < alerts_pool_count; i++ ) + { + alerts_pool[i] = NULL; + } + + alerts_pool_count = 0; + pthread_mutex_unlock ( &alerts_pool_mutex ); + _dpd.logMsg ( "**** UNLOCKED ****\n\n" ); + } + + pthread_exit ((void*) 0); + return (void*) 0; +} /* ----- end of function _AI_serializer_thread ----- */ + + +/** + * \brief Thread for managing the buffer of alerts and serialize them at constant intervals + */ + +PRIVATE void* +_AI_alerts_pool_thread ( void *arg ) +{ + pthread_t serializer_thread; + + while ( 1 ) + { + if ( !conf ) + { + pthread_exit ((void*) 0); + return (void*) 0; + } + + sleep ( conf->alertSerializationInterval ); + + if ( !alerts_pool || alerts_pool_count == 0 ) + continue; + + if ( pthread_create ( &serializer_thread, NULL, _AI_serializer_thread, NULL ) != 0 ) + _dpd.fatalMsg ( "Failed to create the alerts' serializer thread\n" ); + } + + pthread_exit ((void*) 0); + return (void*) 0; +} /* ----- end of function _AI_alerts_pool_thread ----- */ + + /** * \brief Thread for parsing Snort's alert file * \param arg void* pointer to module's configuration @@ -54,14 +132,14 @@ AI_file_alertparser_thread ( void* arg ) }; int i; -#ifndef __APPLE__ +#ifdef LINUX int ifd; int wd; struct stat st; #else - int fd = -1; - struct stat stats; - time_t last_mod_time = (time_t)0; + int fd = -1; + struct stat stats; + time_t last_mod_time = (time_t) 0; #endif int nmatches = 0; char line[8192]; @@ -73,16 +151,34 @@ AI_file_alertparser_thread ( void* arg ) struct pkt_key key; struct pkt_info *info; - AI_config *conf = ( AI_config* ) arg; - AI_snort_alert *alert = NULL; - AI_snort_alert *tmp = NULL; - BOOL in_alert = false; + AI_snort_alert *alert = NULL; + AI_snort_alert *tmp = NULL; + BOOL in_alert = false; + pthread_t alerts_pool_thread; + pthread_t serializer_thread; - pthread_mutex_init ( &mutex, NULL ); + conf = ( AI_config* ) arg; + + /* Initialize the mutex lock, so nobody can read the alerts while we write there */ + pthread_mutex_init ( &alert_mutex, NULL ); + + /* Initialize the mutex on the alerts' pool, so that an only thread per time can write there */ + pthread_mutex_init ( &alerts_pool_mutex, NULL ); + + /* Initialize the pool of alerts to be passed to the serialization thread */ + if ( !( alerts_pool = ( AI_snort_alert** ) malloc ( conf->alert_bufsize * sizeof ( AI_snort_alert* )))) + _dpd.fatalMsg ( "Dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ ); + + for ( i=0; i < conf->alert_bufsize; i++ ) + alerts_pool[i] = NULL; + + /* Initialize the thread for managing the serialization of alerts' pool */ + if ( pthread_create ( &alerts_pool_thread, NULL, _AI_alerts_pool_thread, NULL ) != 0 ) + _dpd.fatalMsg ( "Failed to create the alerts' pool management thread\n" ); while ( 1 ) { -#ifndef MACOS +#ifdef LINUX if (( ifd = inotify_init() ) < 0 ) { _dpd.fatalMsg ( "Could not initialize an inotify object on the alert log file" ); @@ -197,7 +293,8 @@ AI_file_alertparser_thread ( void* arg ) tmp->next = alert; } - /* TODO Do something!! */ + if ( pthread_create ( &serializer_thread, NULL, _AI_serializer_thread, alert ) != 0 ) + _dpd.fatalMsg ( "Failed to create the alerts' serializer thread\n" ); in_alert = false; alert = NULL; @@ -208,7 +305,7 @@ AI_file_alertparser_thread ( void* arg ) if ( !in_alert ) { - pthread_mutex_lock ( &mutex ); + pthread_mutex_lock ( &alert_mutex ); if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 ) { @@ -356,11 +453,12 @@ AI_file_alertparser_thread ( void* arg ) } } - pthread_mutex_unlock ( &mutex ); + pthread_mutex_unlock ( &alert_mutex ); /* AI_alert_serialize ( alert, conf ); */ } - pthread_mutex_destroy ( &mutex ); + free ( alerts_pool ); + pthread_mutex_destroy ( &alert_mutex ); pthread_exit ((void*) 0 ); return (void*) 0; } /* ----- end of function AI_file_alertparser_thread ----- */ @@ -406,9 +504,9 @@ AI_get_alerts () { AI_snort_alert *alerts_copy; - pthread_mutex_lock ( &mutex ); + pthread_mutex_lock ( &alert_mutex ); alerts_copy = _AI_copy_alerts ( alerts ); - pthread_mutex_unlock ( &mutex ); + pthread_mutex_unlock ( &alert_mutex ); return alerts_copy; } /* ----- end of function AI_get_alerts ----- */ diff --git a/spp_ai.c b/spp_ai.c index 2120f95..7ba4736 100644 --- a/spp_ai.c +++ b/spp_ai.c @@ -85,7 +85,7 @@ static void AI_init(char *args) { ex_config = sfPolicyConfigCreate(); if (ex_config == NULL) - _dpd.fatalMsg("Could not allocate configuration struct.\n"); + _dpd.fatalMsg ("Could not allocate configuration struct.\n"); } config = AI_parse(args); @@ -158,16 +158,18 @@ static AI_config * AI_parse(char *args) hierarchy_node **hierarchy_nodes = NULL; int n_hierarchy_nodes = 0; - unsigned long cleanup_interval = 0, - stream_expire_interval = 0, - alertfile_len = 0, - alert_history_file_len = 0, - clusterfile_len = 0, - corr_rules_dir_len = 0, - corr_alerts_dir_len = 0, - alert_clustering_interval = 0, - database_parsing_interval = 0, - correlation_graph_interval = 0; + unsigned long cleanup_interval = 0, + stream_expire_interval = 0, + alertfile_len = 0, + alert_history_file_len = 0, + alert_serialization_interval = 0, + alert_bufsize = 0, + clusterfile_len = 0, + corr_rules_dir_len = 0, + corr_alerts_dir_len = 0, + alert_clustering_interval = 0, + database_parsing_interval = 0, + correlation_graph_interval = 0; BOOL has_cleanup_interval = false, has_stream_expire_interval = false, @@ -285,6 +287,42 @@ static AI_config * AI_parse(char *args) _dpd.logMsg(" Correlation graph thread interval: %d\n", config->correlationGraphInterval); } + /* Parsing the alert_serialization_interval option */ + if (( arg = (char*) strcasestr( args, "alert_serialization_interval" ) )) + { + for ( arg += strlen("alert_serialization_interval"); + *arg && (*arg < '0' || *arg > '9'); + arg++ ); + + if ( !(*arg) ) + { + _dpd.fatalMsg("AIPreproc: alert_serialization_interval option used but " + "no value specified\n"); + } + + alert_serialization_interval = strtoul(arg, NULL, 10); + config->alertSerializationInterval = alert_serialization_interval; + _dpd.logMsg(" Alert serialization thread interval: %d\n", config->correlationGraphInterval); + } + + /* Parsing the alert_bufsize option */ + if (( arg = (char*) strcasestr( args, "alert_bufsize" ) )) + { + for ( arg += strlen("alert_bufsize"); + *arg && (*arg < '0' || *arg > '9'); + arg++ ); + + if ( !(*arg) ) + { + _dpd.fatalMsg("AIPreproc: alert_bufsize option used but " + "no value specified\n"); + } + + alert_bufsize = strtoul(arg, NULL, 10); + config->alert_bufsize= alert_bufsize; + _dpd.logMsg(" Alert buffer size: %d\n", config->alert_bufsize ); + } + /* Parsing the correlation_threshold_coefficient option */ if (( arg = (char*) strcasestr( args, "correlation_threshold_coefficient" ) )) { @@ -849,6 +887,16 @@ static AI_config * AI_parse(char *args) strncpy ( config->corr_alerts_dir, DEFAULT_CORR_ALERTS_DIR, sizeof ( DEFAULT_CORR_ALERTS_DIR )); } + if ( ! alert_serialization_interval ) + { + config->alertSerializationInterval = DEFAULT_ALERT_SERIALIZATION_INTERVAL; + } + + if ( ! alert_bufsize ) + { + config->alert_bufsize = DEFAULT_ALERT_BUFSIZE; + } + _dpd.logMsg ( "Saving correlated alerts information in %s\n", config->corr_alerts_dir ); if ( has_database_log ) diff --git a/spp_ai.h b/spp_ai.h index e097b5e..d8312c2 100644 --- a/spp_ai.h +++ b/spp_ai.h @@ -63,6 +63,12 @@ /** Default correlation threshold coefficient for correlating two hyperalerts */ #define DEFAULT_CORR_THRESHOLD 0.5 +/** Default size of the alerts' buffer to be periodically sent to the serialization thread */ +#define DEFAULT_ALERT_BUFSIZE 30 + +/** Default timeout in seconds between a serialization of the alerts' buffer and the next one */ +#define DEFAULT_ALERT_SERIALIZATION_INTERVAL 3600 + /****************************/ /* Database support */ #ifdef HAVE_LIBMYSQLCLIENT @@ -133,6 +139,12 @@ typedef struct /** Interval in seconds for running the thread for building alert correlation graphs */ unsigned long correlationGraphInterval; + + /** Interval in seconds between a serialization of the alerts' buffer and the next one */ + unsigned long alertSerializationInterval; + + /** Size of the alerts' buffer to be periodically sent to the serialization thread */ + unsigned long alert_bufsize; /** Correlation threshold coefficient for correlating two hyperalerts. Two hyperalerts * are 'correlated' to each other in a multi-step attack graph if and only if their @@ -289,8 +301,8 @@ typedef struct _AI_snort_alert { /*****************************************************************/ int preg_match ( const char*, char*, char***, int* ); -char* str_replace ( char *str, char *orig, char *rep ); -char* str_replace_all ( char *str, char *orig, char *rep ); +char* str_replace ( char*, char*, char *); +char* str_replace_all ( char*, char*, char* ); void* AI_hashcleanup_thread ( void* ); void* AI_file_alertparser_thread ( void* ); @@ -298,7 +310,7 @@ void* AI_alert_correlation_thread ( void* ); #ifdef HAVE_DB AI_snort_alert* AI_db_get_alerts ( void ); -void AI_db_free_alerts ( AI_snort_alert *node ); +void AI_db_free_alerts ( AI_snort_alert* ); void* AI_db_alertparser_thread ( void* ); #endif @@ -311,8 +323,8 @@ struct pkt_info* AI_get_stream_by_key ( struct pkt_key ); AI_snort_alert* AI_get_alerts ( void ); AI_snort_alert* AI_get_clustered_alerts ( void ); -void AI_serialize_alert ( AI_snort_alert*, AI_config* ); -void AI_deserialize_alert ( AI_snort_alert*, AI_config* ); +void AI_serialize_alerts ( AI_snort_alert**, unsigned int, AI_config* ); +void* AI_deserialize_alerts ( AI_config* ); /** Function pointer to the function used for getting the alert list (from log file, db, ...) */ extern AI_snort_alert* (*get_alerts)(void);