blacklight
/
Snort_AIPreproc
mirror of https://github.com/BlackLight/Snort_AIPreproc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOM neural network support for alert correlation

This commit is contained in:
BlackLight 2010-10-25 17:39:44 +02:00
parent 8090600f65
commit e17bbfd91e
8 changed files with 325 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
TODO
View File

@ -2,8 +2,7 @@
AVERAGE/HIGH PRIORITY:
======================
- Neural network for alert correlation
- Modules for correlation coefficients
- Supporting extra modules for alert correlation
- Code profiling
- Comment all the code!!!
- Neural network for computing k
@ -36,4 +35,5 @@ DONE:
+ Function names (private functions with _ or __ ?)
+ Saving packet flows as .pcap
+ Manual alert correlation from the web interface
+ Neural network for alert correlation

2
bayesian.c
View File

@ -77,7 +77,7 @@ __AI_bayesian_correlation_function ( time_t ta, time_t tb )
*/
double
AI_alert_bayesian_correlation ( AI_snort_alert *a, AI_snort_alert *b )
AI_alert_bayesian_correlation ( const AI_snort_alert *a, const AI_snort_alert *b )
{
double corr = 0.0;
unsigned int corr_count = 0,

38
correlation.c
View File

@ -1227,8 +1227,10 @@ AI_alert_correlation_thread ( void *arg )
std_deviation = 0.0,
corr_threshold = 0.0,
kb_correlation = 0.0,
bayesian_correlation = 0.0;
bayesian_correlation = 0.0,
neural_correlation = 0.0;
size_t n_correlations = 0;
FILE *fp = NULL;
AI_alert_correlation_key corr_key;
@ -1348,17 +1350,37 @@ AI_alert_correlation_thread ( void *arg )
corr_key.a = alert_iterator;
corr_key.b = alert_iterator2;
corr->key = corr_key;
corr->correlation = 0.0;
n_correlations = 0;
kb_correlation = __AI_kb_correlation_coefficient ( corr_key.a, corr_key.b );
bayesian_correlation = AI_alert_bayesian_correlation ( corr_key.a, corr_key.b );
neural_correlation = AI_alert_neural_som_correlation ( corr_key.a, corr_key.b );
if ( bayesian_correlation == 0.0 || config->bayesianCorrelationInterval == 0 )
corr->correlation = kb_correlation;
else if ( kb_correlation == 0.0 )
corr->correlation = bayesian_correlation;
else
corr->correlation = ( kb_correlation + bayesian_correlation ) / 2;
/* Use the correlation indexes for which we have a value */
if ( bayesian_correlation != 0.0 && config->bayesianCorrelationInterval != 0 )
{
corr->correlation += bayesian_correlation;
n_correlations++;
}
if ( kb_correlation != 0.0 )
{
corr->correlation += kb_correlation;
n_correlations++;
}
if ( neural_correlation != 0.0 && config->neuralNetworkTrainingInterval != 0 )
{
corr->correlation += neural_correlation;
n_correlations++;
}
if ( n_correlations != 0 )
{
corr->correlation /= (double) n_correlations;
}
HASH_ADD ( hh, correlation_table, key, sizeof ( AI_alert_correlation_key ), corr );
}

4
fsom/fsom.c
View File

@ -630,7 +630,7 @@ som_train_iteration ( som_network_t *net, double *data, size_t iter )
* \param n_data Number of vectors in the input set
*/
static void
void
som_init_weights ( som_network_t *net, double **data, size_t n_data )
{
size_t i = 0,
@ -831,8 +831,6 @@ som_train ( som_network_t *net, double **data, size_t n_data, size_t iter )
x = 0,
y = 0;
som_init_weights ( net, data, n_data );
for ( n=0; n < n_data; n++ )
{
for ( k=1; k <= iter; k++ )

1
fsom/fsom.h
View File

@ -59,6 +59,7 @@ void som_network_destroy ( som_network_t* );
void som_set_inputs ( som_network_t*, double* );
void som_train ( som_network_t*, double**, size_t, size_t );
void som_serialize ( som_network_t*, const char* );
void som_init_weights ( som_network_t*, double**, size_t );
double som_get_best_neuron_coordinates ( som_network_t*, size_t*, size_t* );
som_network_t* som_deserialize ( const char* );
som_network_t* som_network_new ( size_t, size_t, size_t );

258
neural.c
View File

@ -29,31 +29,167 @@
#include <alloca.h>
#include <limits.h>
#include <math.h>
#include <pthread.h>
#include <stdio.h>
#include <sys/stat.h>
#include <time.h>
#include <unistd.h>
enum { som_src_ip, som_dst_ip, som_src_port, som_dst_port, som_time, som_alert_id, SOM_NUM_ITEMS };
/** Enumeration for the input fields of the SOM neural network */
enum { som_src_ip, som_dst_ip, som_src_port, som_dst_port, som_time, som_gid, som_sid, som_rev, SOM_NUM_ITEMS };
PRIVATE time_t latest_serialization_time = ( time_t ) 0;
PRIVATE som_network_t *net = NULL;
typedef struct {
unsigned int gid;
unsigned int sid;
unsigned int rev;
uint32_t src_ip_addr;
uint32_t dst_ip_addr;
uint16_t src_port;
uint16_t dst_port;
time_t timestamp;
} AI_som_alert_tuple;
PRIVATE time_t latest_serialization_time = ( time_t ) 0;
PRIVATE som_network_t *net = NULL;
PRIVATE pthread_mutex_t neural_mutex;
/**
* \brief Convert an alert row fetched from db to a vector suitable for being elaborated by the SOM neural network
* \param alert AI_som_alert_tuple object identifying the alert tuple
* \param data Reference to the vector that will contain the SOM data
*/
PRIVATE void
__AI_alert_to_som_data ( const AI_som_alert_tuple alert, double **input )
{
(*input)[som_gid] = (double) alert.gid / (double) USHRT_MAX;
(*input)[som_sid] = (double) alert.sid / (double) USHRT_MAX;
(*input)[som_rev] = (double) alert.rev / (double) USHRT_MAX;
(*input)[som_time] = (double) alert.timestamp / (double) INT_MAX;
(*input)[som_src_ip] = (double) alert.src_ip_addr / (double) UINT_MAX;
(*input)[som_dst_ip] = (double) alert.dst_ip_addr / (double) UINT_MAX;
(*input)[som_src_port] = (double) alert.src_port / (double) USHRT_MAX;
(*input)[som_dst_port] = (double) alert.dst_port / (double) USHRT_MAX;
} /* ----- end of function __AI_alert_to_som_data ----- */
/**
* \brief Get the distance between two alerts mapped on the SOM neural network
* \param alert1 Tuple identifying the first alert
* \param alert2 Tuple identifying the second alert
* \return The distance between the alerts
*/
PRIVATE double
__AI_som_alert_distance ( const AI_som_alert_tuple alert1, const AI_som_alert_tuple alert2 )
{
double *input1 = NULL,
*input2 = NULL;
size_t x1 = 0,
y1 = 0,
x2 = 0,
y2 = 0;
if ( !( input1 = (double*) alloca ( SOM_NUM_ITEMS * sizeof ( double ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
if ( !( input2 = (double*) alloca ( SOM_NUM_ITEMS * sizeof ( double ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
pthread_mutex_lock ( &neural_mutex );
if ( !net )
{
pthread_mutex_unlock ( &neural_mutex );
return 0.0;
}
__AI_alert_to_som_data ( alert1, &input1 );
som_set_inputs ( net, input1 );
som_get_best_neuron_coordinates ( net, &x1, &y1 );
__AI_alert_to_som_data ( alert2, &input2 );
som_set_inputs ( net, input2 );
som_get_best_neuron_coordinates ( net, &x2, &y2 );
pthread_mutex_unlock ( &neural_mutex );
/* Return the normalized euclidean distance in [0,1] (the normalization is made considering that the maximum distance
* between two points on the output neurons matrix is the distance between the upper-left and bottom-right points) */
return sqrt ((double) ( (x2-x1)*(x2-x1) + (y2-y1)*(y2-y1) )) /
sqrt ((double) ( 2 * (config->outputNeuronsPerSide-1) * (config->outputNeuronsPerSide-1) ));
} /* ----- end of function __AI_som_alert_distance ----- */
/**
* \brief Get the SOM neural correlation between two alerts given as AI_snort_alert objects
* \param a First alert
* \param b Second alert
* \return The correlation between a and b computed by the neural network
*/
double
AI_alert_neural_som_correlation ( const AI_snort_alert *a, const AI_snort_alert *b )
{
size_t i = 0;
unsigned long long int time_sum = 0;
AI_som_alert_tuple t1, t2;
t1.gid = a->gid;
t1.sid = a->sid;
t1.rev = a->rev;
t1.src_ip_addr = ntohl ( a->ip_src_addr );
t1.dst_ip_addr = ntohl ( a->ip_dst_addr );
t1.src_port = ntohs ( a->tcp_src_port );
t1.dst_port = ntohs ( a->tcp_dst_port );
time_sum = (unsigned long long int) a->timestamp;
/* The timestamp of this alert is computed like the average timestamp of the grouped alerts */
for ( i=1; i < a->grouped_alerts_count; i++ )
{
time_sum += (unsigned long long int) a->grouped_alerts[i-1]->timestamp;
}
t1.timestamp = (time_t) ( time_sum / a->grouped_alerts_count );
t2.gid = b->gid;
t2.sid = b->sid;
t2.rev = b->rev;
t2.src_ip_addr = ntohl ( b->ip_src_addr );
t2.dst_ip_addr = ntohl ( b->ip_dst_addr );
t2.src_port = ntohs ( b->tcp_src_port );
t2.dst_port = ntohs ( b->tcp_dst_port );
time_sum = (unsigned long long int) b->timestamp;
for ( i=1; i < b->grouped_alerts_count; i++ )
{
time_sum += (unsigned long long int) b->grouped_alerts[i-1]->timestamp;
}
t2.timestamp = (time_t) ( time_sum / b->grouped_alerts_count );
return __AI_som_alert_distance ( t1, t2 );
} /* ----- end of function AI_alert_neural_som_correlation ----- */
/**
* \brief Train the neural network taking the alerts from the latest serialization time
*/
PRIVATE void
AI_som_train ()
__AI_som_train ()
{
unsigned long snort_id = 0;
double **inputs;
char query[1024] = { 0 };
size_t i = 0,
num_rows = 0;
double **inputs = NULL;
char query[1024] = { 0 };
size_t i = 0,
num_rows = 0;
DB_result res;
DB_row row;
AI_som_alert_tuple *tuples = NULL;
if ( !DB_out_init() )
{
@ -62,19 +198,19 @@ AI_som_train ()
#ifdef HAVE_LIBMYSQLCLIENT
snprintf ( query, sizeof ( query ),
"SELECT gid, sid, rev, timestamp, ip_src_addr, ip_dst_addr, tcp_src_port, tcp_dst_port "
"FROM %s a JOIN %s ip JOIN %s tcp "
"ON a.ip_hdr=ip.ip_hdr_id AND a.tcp_hdr=tcp.tcp_hdr_id "
"WHERE unix_timestamp(timestamp) > %lu",
"SELECT gid, sid, rev, unix_timestamp(timestamp), ip_src_addr, ip_dst_addr, tcp_src_port, tcp_dst_port "
"FROM (%s a LEFT JOIN %s ip ON a.ip_hdr=ip.ip_hdr_id) LEFT JOIN %s tcp "
"ON a.tcp_hdr=tcp.tcp_hdr_id "
"WHERE unix_timestamp(timestamp) >= %lu",
outdb_config[ALERTS_TABLE], outdb_config[IPV4_HEADERS_TABLE], outdb_config[TCP_HEADERS_TABLE],
latest_serialization_time
);
#elif HAVE_LIBPQ
snprintf ( query, sizeof ( query ),
"SELECT gid, sid, rev, timestamp, ip_src_addr, ip_dst_addr, tcp_src_port, tcp_dst_port "
"FROM %s a JOIN %s ip JOIN %s tcp "
"ON a.ip_hdr=ip.ip_hdr_id AND a.tcp_hdr=tcp.tcp_hdr_id "
"WHERE date_part ('epoch', \"timestamp\"(timestamp)) > %lu",
"SELECT gid, sid, rev, date_part('epoch', \"timestamp\"(timestamp)), ip_src_addr, ip_dst_addr, tcp_src_port, tcp_dst_port "
"FROM (%s a LEFT JOIN %s ip ON a.ip_hdr=ip.ip_hdr_id) LEFT JOIN %s tcp "
"ON a.tcp_hdr=tcp.tcp_hdr_id "
"WHERE date_part ('epoch', \"timestamp\"(timestamp)) >= %lu",
outdb_config[ALERTS_TABLE], outdb_config[IPV4_HEADERS_TABLE], outdb_config[TCP_HEADERS_TABLE],
latest_serialization_time
);
@ -87,32 +223,67 @@ AI_som_train ()
num_rows = DB_num_rows ( res );
if ( num_rows == 0 )
{
DB_free_result ( res );
latest_serialization_time = time ( NULL );
return;
}
if ( !( inputs = (double**) alloca ( num_rows * sizeof ( double* ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
if ( !( tuples = (AI_som_alert_tuple*) alloca ( num_rows * sizeof ( AI_som_alert_tuple ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
for ( i=0; i < num_rows; i++ )
{
row = (DB_row) DB_fetch_row ( res );
snort_id = 0;
tuples[i].gid = row[0] ? strtoul ( row[0], NULL, 10 ) : 0;
tuples[i].sid = row[1] ? strtoul ( row[1], NULL, 10 ) : 0;
tuples[i].rev = row[2] ? strtoul ( row[2], NULL, 10 ) : 0;
tuples[i].timestamp = row[3] ? (time_t) strtol ( row[3], NULL, 10 ) : (time_t) 0;
tuples[i].src_ip_addr = row[4] ? ntohl ( inet_addr ( row[4] )) : 0;
tuples[i].dst_ip_addr = row[5] ? ntohl ( inet_addr ( row[5] )) : 0;
tuples[i].src_port = row[6] ? (uint16_t) strtoul ( row[6], NULL, 10 ) : 0;
tuples[i].dst_port = row[7] ? (uint16_t) strtoul ( row[7], NULL, 10 ) : 0;
if ( !( inputs[i] = (double*) alloca ( SOM_NUM_ITEMS * sizeof ( double ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
snort_id = (( strtoul ( row[0], NULL, 10 ) & 0xFFFF ) << 16 ) | ( strtoul ( row[1], NULL, 10 ) & 0xFFFF );
inputs[i][som_alert_id] = (double) snort_id / (double) UINT_MAX;
inputs[i][som_time] = (double) strtol ( row[3], NULL, 10 ) / (double) INT_MAX;
inputs[i][som_src_ip] = (double) ntohl ( inet_addr ( row[4] )) / (double) UINT_MAX;
inputs[i][som_dst_ip] = (double) ntohl ( inet_addr ( row[5] )) / (double) UINT_MAX;
inputs[i][som_src_port] = (double) strtol ( row[6], NULL, 10 ) / (double) USHRT_MAX;
inputs[i][som_dst_port] = (double) strtol ( row[7], NULL, 10 ) / (double) USHRT_MAX;
__AI_alert_to_som_data ( tuples[i], &inputs[i] );
}
DB_free_result ( res );
} /* ----- end of function AI_som_train ----- */
pthread_mutex_lock ( &neural_mutex );
if ( !net )
{
if ( !( net = som_network_new ( SOM_NUM_ITEMS, config->outputNeuronsPerSide, config->outputNeuronsPerSide )))
{
AI_fatal_err ( "AIPreproc: Could not create the neural network", __FILE__, __LINE__ );
}
som_init_weights ( net, inputs, num_rows );
som_train ( net, inputs, num_rows, config->neural_train_steps );
} else {
som_train ( net, inputs, num_rows, config->neural_train_steps );
}
pthread_mutex_unlock ( &neural_mutex );
latest_serialization_time = time ( NULL );
net->serialization_time = latest_serialization_time;
som_serialize ( net, config->netfile );
} /* ----- end of function __AI_som_train ----- */
/**
* \brief Thread for managing the self-organazing map (SOM) neural network for the alert correlation
@ -122,9 +293,10 @@ void*
AI_neural_thread ( void *arg )
{
BOOL do_train = false;
FILE *fp = NULL;
struct stat st;
pthread_mutex_init ( &neural_mutex, NULL );
if ( !config->netfile )
{
AI_fatal_err ( "AIPreproc: neural network thread launched but netfile option was not specified", __FILE__, __LINE__ );
@ -140,39 +312,25 @@ AI_neural_thread ( void *arg )
if ( stat ( config->netfile, &st ) < 0 )
{
do_train = true;
}
if ( !do_train )
{
if ( !( fp = fopen ( config->netfile, "r" )))
} else {
if ( !( net = som_deserialize ( config->netfile )))
{
AI_fatal_err ( "AIPreproc: The neural network file exists but it is not readable", __FILE__, __LINE__ );
AI_fatal_err ( "AIPreproc: Error in deserializing the neural network from the network file", __FILE__, __LINE__ );
}
fread ( &latest_serialization_time, sizeof ( time_t ), 1, fp );
/* If more than N seconds passed from the latest serialization, re-train the neural network */
if ( (int) ( time (NULL) - latest_serialization_time ) > config->neuralNetworkTrainingInterval )
if ( (int) ( time (NULL) - net->serialization_time ) > config->neuralNetworkTrainingInterval )
{
do_train = true;
}
fclose ( fp );
}
if ( !do_train )
if ( do_train )
{
if ( !net )
{
if ( !( net = som_deserialize ( config->netfile )))
{
AI_fatal_err ( "AIPreproc: Error in deserializing the neural network from the network file", __FILE__, __LINE__ );
}
}
sleep ( 5 );
continue;
__AI_som_train();
}
sleep ( config->neuralNetworkTrainingInterval );
}
pthread_exit ((void*) 0);

67
spp_ai.c
View File

@ -196,26 +196,29 @@ static AI_config * AI_parse(char *args)
int n_hierarchy_nodes = 0;
unsigned short webserv_port = 0;
unsigned long cleanup_interval = 0,
stream_expire_interval = 0,
alertfile_len = 0,
unsigned long alertfile_len = 0,
alert_bufsize = 0,
alert_clustering_interval = 0,
alert_correlation_weight = 0,
alert_history_file_len = 0,
alert_serialization_interval = 0,
alert_bufsize = 0,
bayesian_correlation_interval = 0,
bayesian_correlation_cache_validity = 0,
bayesian_correlation_interval = 0,
cleanup_interval = 0,
clusterfile_len = 0,
cluster_max_alert_interval = 0,
corr_rules_dir_len = 0,
corr_alerts_dir_len = 0,
webserv_dir_len = 0,
webserv_banner_len = 0,
alert_clustering_interval = 0,
database_parsing_interval = 0,
corr_rules_dir_len = 0,
correlation_graph_interval = 0,
database_parsing_interval = 0,
manual_correlations_parsing_interval = 0,
neural_network_training_interval = 0,
output_neurons_per_side = 0;
neural_train_steps = 0,
output_neurons_per_side = 0,
stream_expire_interval = 0,
webserv_banner_len = 0,
webserv_dir_len = 0;
BOOL has_cleanup_interval = false,
has_stream_expire_interval = false,
@ -539,6 +542,48 @@ static AI_config * AI_parse(char *args)
config->outputNeuronsPerSide = output_neurons_per_side;
_dpd.logMsg( " Output neurons per side: %u\n", config->outputNeuronsPerSide );
/* Parsing the neural_train_steps option */
if (( arg = (char*) strcasestr( args, "neural_train_steps" ) ))
{
for ( arg += strlen("neural_train_steps");
*arg && (*arg < '0' || *arg > '9');
arg++ );
if ( !(*arg) )
{
AI_fatal_err ( "neural_train_steps option used but "
"no value specified", __FILE__, __LINE__ );
}
neural_train_steps = strtoul ( arg, NULL, 10 );
} else {
neural_train_steps = DEFAULT_NEURAL_TRAIN_STEPS;
}
config->neural_train_steps = neural_train_steps;
_dpd.logMsg( " Neural train steps: %u\n", config->neural_train_steps );
/* Parsing the alert_correlation_weight option */
if (( arg = (char*) strcasestr( args, "alert_correlation_weight" ) ))
{
for ( arg += strlen("alert_correlation_weight");
*arg && (*arg < '0' || *arg > '9');
arg++ );
if ( !(*arg) )
{
AI_fatal_err ( "alert_correlation_weight option used but "
"no value specified", __FILE__, __LINE__ );
}
alert_correlation_weight = strtoul ( arg, NULL, 10 );
} else {
alert_correlation_weight = DEFAULT_ALERT_CORRELATION_WEIGHT;
}
config->alert_correlation_weight = alert_correlation_weight;
_dpd.logMsg( " Alert correlation weight: %u\n", config->alert_correlation_weight );
/* Parsing the alertfile option */
if (( arg = (char*) strcasestr( args, "alertfile" ) ))
{

27
spp_ai.h
View File

@ -88,6 +88,14 @@
/** Default number of neurons per side on the output matrix of the SOM neural network */
#define DEFAULT_OUTPUT_NEURONS_PER_SIDE 20
/** Default number of steps used for training the neural network */
#define DEFAULT_NEURAL_TRAIN_STEPS 10
/** Default number of alerts needed in the history file or database for letting a certain
* heuristic correlation index weight be =~ 0.95 (the weight monotonically increases
* with the number of alerts according to a hyperbolic tangent function) */
#define DEFAULT_ALERT_CORRELATION_WEIGHT 400
/** Default web server port */
#define DEFAULT_WEBSERV_PORT 7654
@ -190,6 +198,14 @@ typedef struct
/** Number of neurons per side on the output matrix of the SOM neural network */
unsigned long outputNeuronsPerSide;
/** Number of alerts needed in the history file or database for letting a certain
* heuristic correlation index weight be =~ 0.95 (the weight monotonically increases
* with the number of alerts according to a hyperbolic tangent function) */
unsigned long alert_correlation_weight;
/** Number of steps used for training the neural network */
unsigned long neural_train_steps;
/** Size of the alerts' buffer to be periodically sent to the serialization thread */
unsigned long alert_bufsize;
@ -427,6 +443,7 @@ typedef struct {
UT_hash_handle hh;
} AI_alert_correlation;
/*****************************************************************/
/** Enumeration for describing the table in the output database */
enum { ALERTS_TABLE, IPV4_HEADERS_TABLE, TCP_HEADERS_TABLE, PACKET_STREAMS_TABLE, CLUSTERED_ALERTS_TABLE, CORRELATED_ALERTS_TABLE, N_TABLES };
@ -435,6 +452,13 @@ static const char *outdb_config[] __attribute__ (( unused )) = {
"ca_alerts", "ca_ipv4_headers", "ca_tcp_headers",
"ca_packet_streams", "ca_clustered_alerts", "ca_correlated_alerts"
};
/*
* The unused attribute is needed for gcc to avoid raising a warning
* of "unused variable" when compiling with -Wall -pedantic -pedatic errors,
* since this array is declared here but only used in two source files
*/
/*****************************************************************/
int preg_match ( const char*, char*, char***, int* );
@ -471,7 +495,8 @@ void* AI_serializer_thread ( void* );
void* AI_neural_thread ( void* );
const AI_alert_event* AI_get_alert_events_by_key ( AI_alert_event_key );
unsigned int AI_get_history_alert_number ();
double AI_alert_bayesian_correlation ( AI_snort_alert *a, AI_snort_alert *b );
double AI_alert_bayesian_correlation ( const AI_snort_alert*, const AI_snort_alert* );
double AI_alert_neural_som_correlation ( const AI_snort_alert*, const AI_snort_alert* );
void AI_outdb_mutex_initialize ();
void* AI_store_alert_to_db_thread ( void* );