blacklight
/
Snort_AIPreproc
mirror of https://github.com/BlackLight/Snort_AIPreproc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Supporting alert history serialization

This commit is contained in:
BlackLight 2010-09-21 16:27:46 +02:00
parent 93e0ba6511
commit 684f387a6e
7 changed files with 421 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.am
View File

@ -16,6 +16,7 @@ include/sf_dynamic_preproc_lib.c \
include/sfPolicyUserData.c
libsf_ai_preproc_la_SOURCES = \
alert_history.c \
alert_parser.c \
cluster.c \
correlation.c \

7
Makefile.in
View File

@ -75,7 +75,8 @@ am__installdirs = "$(DESTDIR)$(libdir)" "$(DESTDIR)$(corr_rulesdir)" \
"$(DESTDIR)$(sharedir)"
LTLIBRARIES = $(lib_LTLIBRARIES)
libsf_ai_preproc_la_LIBADD =
am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_parser.lo \
am_libsf_ai_preproc_la_OBJECTS = libsf_ai_preproc_la-alert_history.lo \
libsf_ai_preproc_la-alert_parser.lo \
libsf_ai_preproc_la-cluster.lo \
libsf_ai_preproc_la-correlation.lo libsf_ai_preproc_la-db.lo \
libsf_ai_preproc_la-mysql.lo libsf_ai_preproc_la-postgresql.lo \
@ -250,6 +251,7 @@ include/sf_dynamic_preproc_lib.c \
include/sfPolicyUserData.c
libsf_ai_preproc_la_SOURCES = \
alert_history.c \
alert_parser.c \
cluster.c \
correlation.c \
@ -371,6 +373,9 @@ distclean-compile:
.c.lo:
$(LTCOMPILE) -c -o $@ $<
libsf_ai_preproc_la-alert_history.lo: alert_history.c
$(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-alert_history.lo `test -f 'alert_history.c' || echo '$(srcdir)/'`alert_history.c
libsf_ai_preproc_la-alert_parser.lo: alert_parser.c
$(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) $(libsf_ai_preproc_la_CFLAGS) $(CFLAGS) -c -o libsf_ai_preproc_la-alert_parser.lo `test -f 'alert_parser.c' || echo '$(srcdir)/'`alert_parser.c

1
TODO
View File

@ -2,6 +2,7 @@
AVERAGE/HIGH PRIORITY:
======================
- Add alerts' history serialization to db.c as well
- Testing more scenarios, making more hyperalert models
- Bayesian learning among alerts in alert log
- libgc support

220
alert_history.c
View File

@ -19,3 +19,223 @@
#include "spp_ai.h"
#include <sys/stat.h>
typedef struct {
int gid;
int sid;
int rev;
} AI_alert_event_key;
typedef struct _AI_alert_event {
AI_alert_event_key key;
unsigned int count;
time_t timestamp;
struct _AI_alert_event *next;
UT_hash_handle hh;
} AI_alert_event;
PRIVATE AI_alert_event *alerts_hash = NULL;
/**
* FUNCTION: AI_alerts_hash_free
* \brief Free a hash table of alert events
* \param events Hash table to be freed
*/
void
AI_alerts_hash_free ( AI_alert_event **events )
{
AI_alert_event *hash_iterator = NULL,
*list_iterator = NULL,
*tmp = NULL;
while ( *events )
{
hash_iterator = *events;
HASH_DEL ( *events, hash_iterator );
list_iterator = hash_iterator;
while ( list_iterator )
{
tmp = list_iterator->next;
free ( list_iterator );
list_iterator = tmp;
}
free ( hash_iterator );
}
*events = NULL;
} /* ----- end of function AI_alerts_hash_free ----- */
/**
* \brief Deserialize a alerts' hash table from the binary history file
* \param conf Configuration of the module
* \return A void* pointer (to be casted to AI_alert_event*) to the stored hash table
*/
void*
AI_deserialize_alerts ( AI_config *conf )
{
FILE *fp = NULL;
struct stat st;
unsigned int i, j,
lists_count = 0,
items_count = 0;
AI_alert_event *event_iterator = NULL,
*event_prev = NULL,
*event_list = NULL;
AI_alert_event_key key;
if ( stat ( conf->alert_history_file, &st ) < 0 )
return NULL;
if ( ! S_ISREG ( st.st_mode ))
_dpd.fatalMsg ( "AIPreproc: '%s' is not a regular file\n", conf->alert_history_file );
if ( !( fp = fopen ( conf->alert_history_file, "r" )))
_dpd.fatalMsg ( "AIPreproc: Unable to read from the file '%s'\n", conf->alert_history_file );
AI_alerts_hash_free ( &alerts_hash );
if ( fread ( &lists_count, sizeof ( unsigned int ), 1, fp ) <= 0 )
_dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file );
/* Fill the hash table reading from the file */
for ( i=0; i < lists_count; i++ )
{
event_iterator = NULL;
event_prev = NULL;
if ( fread ( &items_count, sizeof ( unsigned int ), 1, fp ) <= 0 )
_dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file );
for ( j=0; j < items_count; j++ )
{
if ( j == 0 )
{
if ( !( event_list = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event ))))
_dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
memset ( event_list, 0, sizeof ( AI_alert_event ));
event_iterator = event_list;
} else {
if ( !( event_iterator = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event ))))
_dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
memset ( event_iterator, 0, sizeof ( AI_alert_event ));
}
event_iterator->count = items_count;
if ( fread ( &( event_iterator->key ), sizeof ( event_iterator->key ), 1, fp ) <= 0 )
_dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file );
if ( fread ( &( event_iterator->timestamp ), sizeof ( event_iterator->timestamp ), 1, fp ) <= 0 )
_dpd.fatalMsg ( "AIPreproc: Malformed history file '%s'\n", conf->alert_history_file );
if ( event_prev )
{
event_prev->next = event_iterator;
}
event_prev = event_iterator;
}
key = event_iterator->key;
HASH_ADD ( hh, alerts_hash, key, sizeof ( key ), event_list );
}
fclose ( fp );
return (void*) alerts_hash;
} /* ----- end of function AI_deserialize_alerts ----- */
/**
* \brief Serialize a buffer of alerts to the binary history file
* \param alerts_pool Buffer of alerts to be serialized
* \param alerts_pool_count Number of alerts in the buffer
* \param conf Configuration of the module
*/
void
AI_serialize_alerts ( AI_snort_alert **alerts_pool, unsigned int alerts_pool_count, AI_config *conf )
{
unsigned int i,
hash_count = 0,
list_count = 0;
FILE *fp = NULL;
AI_alert_event_key key;
AI_alert_event *found = NULL,
*event = NULL,
*event_next = NULL,
*event_iterator = NULL;
if ( !alerts_hash )
{
AI_deserialize_alerts ( conf );
}
for ( i=0; i < alerts_pool_count; i++ )
{
if ( !( event = ( AI_alert_event* ) malloc ( sizeof ( AI_alert_event ))))
_dpd.fatalMsg ( "AIPreproc: Fatal dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
memset ( event, 0, sizeof ( AI_alert_event ));
key.gid = alerts_pool[i]->gid;
key.sid = alerts_pool[i]->sid;
key.rev = alerts_pool[i]->rev;
event->key = key;
event->timestamp = alerts_pool[i]->timestamp;
HASH_FIND ( hh, alerts_hash, &key, sizeof ( key ), found );
if ( !found )
{
event->count = 1;
event->next = NULL;
HASH_ADD ( hh, alerts_hash, key, sizeof ( key ), event );
} else {
found->count++;
event_next = NULL;
for ( event_iterator = found; event_iterator->next; event_iterator = event_iterator->next )
{
/* Insert the new event in cronological order */
if ( event_iterator->next->timestamp > event->timestamp )
{
event_next = event_iterator->next;
break;
}
}
if ( event_iterator )
event_iterator->next = event;
event->next = event_next;
}
}
hash_count = HASH_COUNT ( alerts_hash );
if ( !( fp = fopen ( conf->alert_history_file, "w" )))
_dpd.fatalMsg ( "AIPreproc: Unable to write on '%s'\n", conf->alert_history_file );
fwrite ( &hash_count, sizeof ( hash_count ), 1, fp );
for ( event = alerts_hash; event; event = ( AI_alert_event* ) event->hh.next )
{
list_count = event->count;
fwrite ( &list_count, sizeof ( list_count ), 1, fp );
for ( event_iterator = event; event_iterator; event_iterator = event_iterator->next )
{
fwrite ( &(event_iterator->key), sizeof ( event_iterator->key ), 1, fp );
fwrite ( &(event_iterator->timestamp), sizeof ( event_iterator->timestamp ), 1, fp );
}
}
fclose ( fp );
} /* ----- end of function AI_serialize_alerts ----- */

136
alert_parser.c
View File

@ -29,13 +29,91 @@
#include <pthread.h>
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE FILE *alert_fp = NULL;
PRIVATE pthread_mutex_t mutex;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE AI_snort_alert **alerts_pool = NULL;
PRIVATE AI_config *conf = NULL;
PRIVATE FILE *alert_fp = NULL;
PRIVATE unsigned int alerts_pool_count = 0;
PRIVATE pthread_mutex_t alert_mutex;
PRIVATE pthread_mutex_t alerts_pool_mutex;
/** \defgroup alert_parser Parse the alert log into binary structures
* @{ */
/**
* \brief Serialize the pool of alerts in a separated thread
* \param arg void* pointer to the alert to be added to the pool, if any
*/
PRIVATE void*
_AI_serializer_thread ( void *arg )
{
unsigned int i = 0;
AI_snort_alert *alert = NULL;
if ( arg )
{
alert = ( AI_snort_alert* ) arg;
pthread_mutex_lock ( &alerts_pool_mutex );
alerts_pool [ alerts_pool_count++ ] = alert;
pthread_mutex_unlock ( &alerts_pool_mutex );
}
if ( !arg || ( arg && alerts_pool_count >= conf->alert_bufsize ))
{
pthread_mutex_lock ( &alerts_pool_mutex );
_dpd.logMsg ( "**** LOCKED ****\n" );
AI_serialize_alerts ( alerts_pool, alerts_pool_count, conf );
for ( i=0; i < alerts_pool_count; i++ )
{
alerts_pool[i] = NULL;
}
alerts_pool_count = 0;
pthread_mutex_unlock ( &alerts_pool_mutex );
_dpd.logMsg ( "**** UNLOCKED ****\n\n" );
}
pthread_exit ((void*) 0);
return (void*) 0;
} /* ----- end of function _AI_serializer_thread ----- */
/**
* \brief Thread for managing the buffer of alerts and serialize them at constant intervals
*/
PRIVATE void*
_AI_alerts_pool_thread ( void *arg )
{
pthread_t serializer_thread;
while ( 1 )
{
if ( !conf )
{
pthread_exit ((void*) 0);
return (void*) 0;
}
sleep ( conf->alertSerializationInterval );
if ( !alerts_pool || alerts_pool_count == 0 )
continue;
if ( pthread_create ( &serializer_thread, NULL, _AI_serializer_thread, NULL ) != 0 )
_dpd.fatalMsg ( "Failed to create the alerts' serializer thread\n" );
}
pthread_exit ((void*) 0);
return (void*) 0;
} /* ----- end of function _AI_alerts_pool_thread ----- */
/**
* \brief Thread for parsing Snort's alert file
* \param arg void* pointer to module's configuration
@ -54,14 +132,14 @@ AI_file_alertparser_thread ( void* arg )
};
int i;
#ifndef __APPLE__
#ifdef LINUX
int ifd;
int wd;
struct stat st;
#else
int fd = -1;
struct stat stats;
time_t last_mod_time = (time_t)0;
int fd = -1;
struct stat stats;
time_t last_mod_time = (time_t) 0;
#endif
int nmatches = 0;
char line[8192];
@ -73,16 +151,34 @@ AI_file_alertparser_thread ( void* arg )
struct pkt_key key;
struct pkt_info *info;
AI_config *conf = ( AI_config* ) arg;
AI_snort_alert *alert = NULL;
AI_snort_alert *tmp = NULL;
BOOL in_alert = false;
AI_snort_alert *alert = NULL;
AI_snort_alert *tmp = NULL;
BOOL in_alert = false;
pthread_t alerts_pool_thread;
pthread_t serializer_thread;
pthread_mutex_init ( &mutex, NULL );
conf = ( AI_config* ) arg;
/* Initialize the mutex lock, so nobody can read the alerts while we write there */
pthread_mutex_init ( &alert_mutex, NULL );
/* Initialize the mutex on the alerts' pool, so that an only thread per time can write there */
pthread_mutex_init ( &alerts_pool_mutex, NULL );
/* Initialize the pool of alerts to be passed to the serialization thread */
if ( !( alerts_pool = ( AI_snort_alert** ) malloc ( conf->alert_bufsize * sizeof ( AI_snort_alert* ))))
_dpd.fatalMsg ( "Dynamic memory allocation error at %s:%d\n", __FILE__, __LINE__ );
for ( i=0; i < conf->alert_bufsize; i++ )
alerts_pool[i] = NULL;
/* Initialize the thread for managing the serialization of alerts' pool */
if ( pthread_create ( &alerts_pool_thread, NULL, _AI_alerts_pool_thread, NULL ) != 0 )
_dpd.fatalMsg ( "Failed to create the alerts' pool management thread\n" );
while ( 1 )
{
#ifndef MACOS
#ifdef LINUX
if (( ifd = inotify_init() ) < 0 )
{
_dpd.fatalMsg ( "Could not initialize an inotify object on the alert log file" );
@ -197,7 +293,8 @@ AI_file_alertparser_thread ( void* arg )
tmp->next = alert;
}
/* TODO Do something!! */
if ( pthread_create ( &serializer_thread, NULL, _AI_serializer_thread, alert ) != 0 )
_dpd.fatalMsg ( "Failed to create the alerts' serializer thread\n" );
in_alert = false;
alert = NULL;
@ -208,7 +305,7 @@ AI_file_alertparser_thread ( void* arg )
if ( !in_alert )
{
pthread_mutex_lock ( &mutex );
pthread_mutex_lock ( &alert_mutex );
if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 )
{
@ -356,11 +453,12 @@ AI_file_alertparser_thread ( void* arg )
}
}
pthread_mutex_unlock ( &mutex );
pthread_mutex_unlock ( &alert_mutex );
/* AI_alert_serialize ( alert, conf ); */
}
pthread_mutex_destroy ( &mutex );
free ( alerts_pool );
pthread_mutex_destroy ( &alert_mutex );
pthread_exit ((void*) 0 );
return (void*) 0;
} /* ----- end of function AI_file_alertparser_thread ----- */
@ -406,9 +504,9 @@ AI_get_alerts ()
{
AI_snort_alert *alerts_copy;
pthread_mutex_lock ( &mutex );
pthread_mutex_lock ( &alert_mutex );
alerts_copy = _AI_copy_alerts ( alerts );
pthread_mutex_unlock ( &mutex );
pthread_mutex_unlock ( &alert_mutex );
return alerts_copy;
} /* ----- end of function AI_get_alerts ----- */

70
spp_ai.c
View File

@ -85,7 +85,7 @@ static void AI_init(char *args)
{
ex_config = sfPolicyConfigCreate();
if (ex_config == NULL)
_dpd.fatalMsg("Could not allocate configuration struct.\n");
_dpd.fatalMsg ("Could not allocate configuration struct.\n");
}
config = AI_parse(args);
@ -158,16 +158,18 @@ static AI_config * AI_parse(char *args)
hierarchy_node **hierarchy_nodes = NULL;
int n_hierarchy_nodes = 0;
unsigned long cleanup_interval = 0,
stream_expire_interval = 0,
alertfile_len = 0,
alert_history_file_len = 0,
clusterfile_len = 0,
corr_rules_dir_len = 0,
corr_alerts_dir_len = 0,
alert_clustering_interval = 0,
database_parsing_interval = 0,
correlation_graph_interval = 0;
unsigned long cleanup_interval = 0,
stream_expire_interval = 0,
alertfile_len = 0,
alert_history_file_len = 0,
alert_serialization_interval = 0,
alert_bufsize = 0,
clusterfile_len = 0,
corr_rules_dir_len = 0,
corr_alerts_dir_len = 0,
alert_clustering_interval = 0,
database_parsing_interval = 0,
correlation_graph_interval = 0;
BOOL has_cleanup_interval = false,
has_stream_expire_interval = false,
@ -285,6 +287,42 @@ static AI_config * AI_parse(char *args)
_dpd.logMsg(" Correlation graph thread interval: %d\n", config->correlationGraphInterval);
}
/* Parsing the alert_serialization_interval option */
if (( arg = (char*) strcasestr( args, "alert_serialization_interval" ) ))
{
for ( arg += strlen("alert_serialization_interval");
*arg && (*arg < '0' || *arg > '9');
arg++ );
if ( !(*arg) )
{
_dpd.fatalMsg("AIPreproc: alert_serialization_interval option used but "
"no value specified\n");
}
alert_serialization_interval = strtoul(arg, NULL, 10);
config->alertSerializationInterval = alert_serialization_interval;
_dpd.logMsg(" Alert serialization thread interval: %d\n", config->correlationGraphInterval);
}
/* Parsing the alert_bufsize option */
if (( arg = (char*) strcasestr( args, "alert_bufsize" ) ))
{
for ( arg += strlen("alert_bufsize");
*arg && (*arg < '0' || *arg > '9');
arg++ );
if ( !(*arg) )
{
_dpd.fatalMsg("AIPreproc: alert_bufsize option used but "
"no value specified\n");
}
alert_bufsize = strtoul(arg, NULL, 10);
config->alert_bufsize= alert_bufsize;
_dpd.logMsg(" Alert buffer size: %d\n", config->alert_bufsize );
}
/* Parsing the correlation_threshold_coefficient option */
if (( arg = (char*) strcasestr( args, "correlation_threshold_coefficient" ) ))
{
@ -849,6 +887,16 @@ static AI_config * AI_parse(char *args)
strncpy ( config->corr_alerts_dir, DEFAULT_CORR_ALERTS_DIR, sizeof ( DEFAULT_CORR_ALERTS_DIR ));
}
if ( ! alert_serialization_interval )
{
config->alertSerializationInterval = DEFAULT_ALERT_SERIALIZATION_INTERVAL;
}
if ( ! alert_bufsize )
{
config->alert_bufsize = DEFAULT_ALERT_BUFSIZE;
}
_dpd.logMsg ( "Saving correlated alerts information in %s\n", config->corr_alerts_dir );
if ( has_database_log )

22
spp_ai.h
View File

@ -63,6 +63,12 @@
/** Default correlation threshold coefficient for correlating two hyperalerts */
#define DEFAULT_CORR_THRESHOLD 0.5
/** Default size of the alerts' buffer to be periodically sent to the serialization thread */
#define DEFAULT_ALERT_BUFSIZE 30
/** Default timeout in seconds between a serialization of the alerts' buffer and the next one */
#define DEFAULT_ALERT_SERIALIZATION_INTERVAL 3600
/****************************/
/* Database support */
#ifdef HAVE_LIBMYSQLCLIENT
@ -133,6 +139,12 @@ typedef struct
/** Interval in seconds for running the thread for building alert correlation graphs */
unsigned long correlationGraphInterval;
/** Interval in seconds between a serialization of the alerts' buffer and the next one */
unsigned long alertSerializationInterval;
/** Size of the alerts' buffer to be periodically sent to the serialization thread */
unsigned long alert_bufsize;
/** Correlation threshold coefficient for correlating two hyperalerts. Two hyperalerts
* are 'correlated' to each other in a multi-step attack graph if and only if their
@ -289,8 +301,8 @@ typedef struct _AI_snort_alert {
/*****************************************************************/
int preg_match ( const char*, char*, char***, int* );
char* str_replace ( char *str, char *orig, char *rep );
char* str_replace_all ( char *str, char *orig, char *rep );
char* str_replace ( char*, char*, char *);
char* str_replace_all ( char*, char*, char* );
void* AI_hashcleanup_thread ( void* );
void* AI_file_alertparser_thread ( void* );
@ -298,7 +310,7 @@ void* AI_alert_correlation_thread ( void* );
#ifdef HAVE_DB
AI_snort_alert* AI_db_get_alerts ( void );
void AI_db_free_alerts ( AI_snort_alert *node );
void AI_db_free_alerts ( AI_snort_alert* );
void* AI_db_alertparser_thread ( void* );
#endif
@ -311,8 +323,8 @@ struct pkt_info* AI_get_stream_by_key ( struct pkt_key );
AI_snort_alert* AI_get_alerts ( void );
AI_snort_alert* AI_get_clustered_alerts ( void );
void AI_serialize_alert ( AI_snort_alert*, AI_config* );
void AI_deserialize_alert ( AI_snort_alert*, AI_config* );
void AI_serialize_alerts ( AI_snort_alert**, unsigned int, AI_config* );
void* AI_deserialize_alerts ( AI_config* );
/** Function pointer to the function used for getting the alert list (from log file, db, ...) */
extern AI_snort_alert* (*get_alerts)(void);