mirror of
https://github.com/BlackLight/Snort_AIPreproc.git
synced 2024-11-13 04:07:15 +01:00
Improved multithread locks management
This commit is contained in:
parent
960b70e106
commit
93e0ba6511
9 changed files with 140 additions and 47 deletions
7
README
7
README
|
@ -147,7 +147,8 @@ following:
|
|||
preprocessor ai: \
|
||||
hashtable_cleanup_interval 300 \
|
||||
tcp_stream_expire_interval 300 \
|
||||
alertfile "/home/youruser/local/snort/log/alert" \
|
||||
alertfile "/your/snort/dir/log/alert" \
|
||||
alert_history_file "/your/snort/dir/log/alert_history" \
|
||||
alert_clustering_interval 300 \
|
||||
correlation_graph_interval 300 \
|
||||
correlation_rules_dir "/your/snort/dir/etc/corr_rules" \
|
||||
|
@ -178,6 +179,10 @@ stream as "expired", if no more packets are received inside of that and it's not
|
|||
- alertfile: The file where Snort saves its alerts, if they are saved to a file
|
||||
and not to a database (default if not specified: /var/log/snort/alert)
|
||||
|
||||
- alert_history_file: The file keeping track of the history, in binary format,
|
||||
of all the alerts received by the IDS, so that the module can build some
|
||||
statistical correlation inferences over the past
|
||||
|
||||
- alert_clustering_interval: The interval that should occur from the clustering
|
||||
of the alerts in the log according to the provided clustering hierarchies and
|
||||
the next one (default if not specified: 300 seconds)
|
||||
|
|
1
TODO
1
TODO
|
@ -2,7 +2,6 @@
|
|||
AVERAGE/HIGH PRIORITY:
|
||||
======================
|
||||
|
||||
- Dynamic k parameter in correlation threshold
|
||||
- Testing more scenarios, making more hyperalert models
|
||||
- Bayesian learning among alerts in alert log
|
||||
- libgc support
|
||||
|
|
21
alert_history.c
Normal file
21
alert_history.c
Normal file
|
@ -0,0 +1,21 @@
|
|||
/*
|
||||
* =====================================================================================
|
||||
*
|
||||
* Filename: alert_history.c
|
||||
*
|
||||
* Description: Manages the history of alerts on a binary file
|
||||
*
|
||||
* Version: 0.1
|
||||
* Created: 18/09/2010 21:02:15
|
||||
* Revision: none
|
||||
* Compiler: gcc
|
||||
*
|
||||
* Author: BlackLight (http://0x00.ath.cx), <blacklight@autistici.org>
|
||||
* Licence: GNU GPL v.3
|
||||
* Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE!
|
||||
*
|
||||
* =====================================================================================
|
||||
*/
|
||||
|
||||
#include "spp_ai.h"
|
||||
|
|
@ -29,9 +29,9 @@
|
|||
#include <pthread.h>
|
||||
|
||||
|
||||
PRIVATE AI_snort_alert *alerts = NULL;
|
||||
PRIVATE FILE *alert_fp = NULL;
|
||||
PRIVATE BOOL lock_flag = false;
|
||||
PRIVATE AI_snort_alert *alerts = NULL;
|
||||
PRIVATE FILE *alert_fp = NULL;
|
||||
PRIVATE pthread_mutex_t mutex;
|
||||
|
||||
/** \defgroup alert_parser Parse the alert log into binary structures
|
||||
* @{ */
|
||||
|
@ -78,6 +78,8 @@ AI_file_alertparser_thread ( void* arg )
|
|||
AI_snort_alert *tmp = NULL;
|
||||
BOOL in_alert = false;
|
||||
|
||||
pthread_mutex_init ( &mutex, NULL );
|
||||
|
||||
while ( 1 )
|
||||
{
|
||||
#ifndef MACOS
|
||||
|
@ -132,6 +134,7 @@ AI_file_alertparser_thread ( void* arg )
|
|||
fd = fileno(alert_fp);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Cause the thread to wait until a new file modification (a new alert).
|
||||
*/
|
||||
|
@ -139,6 +142,7 @@ AI_file_alertparser_thread ( void* arg )
|
|||
usleep(100);
|
||||
fstats( fd, &stats );
|
||||
}
|
||||
|
||||
/*
|
||||
* The first time the thread is called, the flow exits instantly from the while,
|
||||
* so this first time the stats structure has to be initialized properly.
|
||||
|
@ -153,7 +157,6 @@ AI_file_alertparser_thread ( void* arg )
|
|||
#endif
|
||||
|
||||
/* Set the lock flag to true until it's done with alert parsing */
|
||||
lock_flag = true;
|
||||
|
||||
while ( !feof ( alert_fp ))
|
||||
{
|
||||
|
@ -205,6 +208,8 @@ AI_file_alertparser_thread ( void* arg )
|
|||
|
||||
if ( !in_alert )
|
||||
{
|
||||
pthread_mutex_lock ( &mutex );
|
||||
|
||||
if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 )
|
||||
{
|
||||
in_alert = true;
|
||||
|
@ -351,9 +356,11 @@ AI_file_alertparser_thread ( void* arg )
|
|||
}
|
||||
}
|
||||
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
/* AI_alert_serialize ( alert, conf ); */
|
||||
}
|
||||
|
||||
pthread_mutex_destroy ( &mutex );
|
||||
pthread_exit ((void*) 0 );
|
||||
return (void*) 0;
|
||||
} /* ----- end of function AI_file_alertparser_thread ----- */
|
||||
|
@ -397,8 +404,13 @@ _AI_copy_alerts ( AI_snort_alert *node )
|
|||
AI_snort_alert*
|
||||
AI_get_alerts ()
|
||||
{
|
||||
while ( lock_flag );
|
||||
return _AI_copy_alerts ( alerts );
|
||||
AI_snort_alert *alerts_copy;
|
||||
|
||||
pthread_mutex_lock ( &mutex );
|
||||
alerts_copy = _AI_copy_alerts ( alerts );
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
|
||||
return alerts_copy;
|
||||
} /* ----- end of function AI_get_alerts ----- */
|
||||
|
||||
|
||||
|
|
33
cluster.c
33
cluster.c
|
@ -50,11 +50,10 @@ typedef struct {
|
|||
} AI_alert_occurrence;
|
||||
|
||||
|
||||
PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL };
|
||||
PRIVATE AI_config *_config = NULL;
|
||||
PRIVATE AI_snort_alert *alert_log = NULL;
|
||||
PRIVATE BOOL lock_flag = false;
|
||||
|
||||
PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL };
|
||||
PRIVATE AI_config *_config = NULL;
|
||||
PRIVATE AI_snort_alert *alert_log = NULL;
|
||||
PRIVATE pthread_mutex_t mutex;
|
||||
|
||||
/**
|
||||
* \brief Function that picks up the heuristic value for a clustering attribute in according to Julisch's heuristic (ACM, Vol.2, No.3, 09 2002, pag.124)
|
||||
|
@ -373,7 +372,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
|
|||
{
|
||||
AI_snort_alert *tmp;
|
||||
char ip[INET_ADDRSTRLEN];
|
||||
char *timestamp;
|
||||
char timestamp[128];
|
||||
struct tm *_tm;
|
||||
|
||||
for ( tmp = log; tmp; tmp = tmp->next )
|
||||
{
|
||||
|
@ -384,8 +384,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
|
|||
|
||||
fprintf ( fp, "[Priority: %d]\n", tmp->priority );
|
||||
|
||||
timestamp = ctime ( &tmp->timestamp );
|
||||
timestamp[ strlen(timestamp)-1 ] = 0;
|
||||
_tm = localtime ( &tmp->timestamp );
|
||||
strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm );
|
||||
fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
|
||||
|
||||
if ( h_root[src_addr] && tmp->h_node[src_addr] )
|
||||
|
@ -445,13 +445,15 @@ _AI_cluster_thread ( void* arg )
|
|||
int single_alerts_count = 0;
|
||||
double heterogeneity = 0;
|
||||
|
||||
pthread_mutex_init ( &mutex, NULL );
|
||||
|
||||
while ( 1 )
|
||||
{
|
||||
/* Between an execution of the thread and the next one, sleep for alert_clustering_interval seconds */
|
||||
sleep ( _config->alertClusteringInterval );
|
||||
|
||||
/* Set the lock over the alert log until it's done with the clustering operation */
|
||||
lock_flag = true;
|
||||
pthread_mutex_lock ( &mutex );
|
||||
|
||||
/* Free the current alert log and get the latest one */
|
||||
if ( alert_log )
|
||||
|
@ -462,7 +464,7 @@ _AI_cluster_thread ( void* arg )
|
|||
|
||||
if ( !( alert_log = get_alerts() ))
|
||||
{
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -561,7 +563,7 @@ _AI_cluster_thread ( void* arg )
|
|||
alert_count -= _AI_merge_alerts ( &alert_log );
|
||||
} while ( old_alert_count != alert_count );
|
||||
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
|
||||
if ( !( cluster_fp = fopen ( _config->clusterfile, "w" )) )
|
||||
{
|
||||
|
@ -732,8 +734,13 @@ _AI_copy_clustered_alerts ( AI_snort_alert *node )
|
|||
AI_snort_alert*
|
||||
AI_get_clustered_alerts ()
|
||||
{
|
||||
for ( ; lock_flag; usleep(100) );
|
||||
return _AI_copy_clustered_alerts ( alert_log );
|
||||
AI_snort_alert *alerts_copy;
|
||||
|
||||
pthread_mutex_lock ( &mutex );
|
||||
alerts_copy = _AI_copy_clustered_alerts ( alert_log );
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
|
||||
return alerts_copy;
|
||||
} /* ----- end of function AI_get_clustered_alerts ----- */
|
||||
|
||||
/** @} */
|
||||
|
|
|
@ -70,7 +70,7 @@ PRIVATE AI_hyperalert_info *hyperalerts = NULL;
|
|||
PRIVATE AI_config *conf = NULL;
|
||||
PRIVATE AI_snort_alert *alerts = NULL;
|
||||
PRIVATE AI_alert_correlation *correlation_table = NULL;
|
||||
PRIVATE BOOL lock_flag = false;
|
||||
PRIVATE pthread_mutex_t mutex;
|
||||
|
||||
/**
|
||||
* \brief Clean up the correlation hash table
|
||||
|
@ -713,6 +713,7 @@ AI_alert_correlation_thread ( void *arg )
|
|||
#endif
|
||||
|
||||
conf = (AI_config*) arg;
|
||||
pthread_mutex_init ( &mutex, NULL );
|
||||
|
||||
while ( 1 )
|
||||
{
|
||||
|
@ -727,7 +728,7 @@ AI_alert_correlation_thread ( void *arg )
|
|||
}
|
||||
|
||||
/* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */
|
||||
lock_flag = true;
|
||||
pthread_mutex_lock ( &mutex );
|
||||
|
||||
if ( alerts )
|
||||
{
|
||||
|
@ -737,7 +738,7 @@ AI_alert_correlation_thread ( void *arg )
|
|||
|
||||
if ( !( alerts = AI_get_clustered_alerts() ))
|
||||
{
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -894,7 +895,7 @@ AI_alert_correlation_thread ( void *arg )
|
|||
#endif
|
||||
}
|
||||
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
}
|
||||
|
||||
pthread_exit (( void* ) 0 );
|
||||
|
|
30
db.c
30
db.c
|
@ -30,12 +30,9 @@
|
|||
* @{ */
|
||||
|
||||
|
||||
PRIVATE AI_config *config;
|
||||
PRIVATE AI_snort_alert *alerts = NULL;
|
||||
PRIVATE BOOL lock_flag = false;
|
||||
|
||||
/** pthread mutex for accessing database data */
|
||||
PRIVATE pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER;
|
||||
PRIVATE AI_config *config;
|
||||
PRIVATE AI_snort_alert *alerts = NULL;
|
||||
PRIVATE pthread_mutex_t mutex;
|
||||
|
||||
/**
|
||||
* \brief Thread for parsing alerts from a database
|
||||
|
@ -65,7 +62,7 @@ AI_db_alertparser_thread ( void *arg )
|
|||
}
|
||||
|
||||
config = ( AI_config* ) arg;
|
||||
pthread_mutex_lock ( &db_mutex );
|
||||
pthread_mutex_init ( &mutex, NULL );
|
||||
|
||||
if ( !DB_init ( config ))
|
||||
{
|
||||
|
@ -73,14 +70,10 @@ AI_db_alertparser_thread ( void *arg )
|
|||
config->dbname, config->dbhost );
|
||||
}
|
||||
|
||||
pthread_mutex_unlock ( &db_mutex );
|
||||
|
||||
while ( 1 )
|
||||
{
|
||||
sleep ( config->databaseParsingInterval );
|
||||
|
||||
/* Set the lock flag to true until it's done with alert parsing */
|
||||
lock_flag = true;
|
||||
pthread_mutex_lock ( &mutex );
|
||||
|
||||
memset ( query, 0, sizeof ( query ));
|
||||
snprintf ( query, sizeof (query), "select cid, unix_timestamp(timestamp), signature from event where cid > %d "
|
||||
|
@ -98,7 +91,7 @@ AI_db_alertparser_thread ( void *arg )
|
|||
DB_close();
|
||||
_dpd.fatalMsg ( "AIPreproc: Could not store the query result at %s:%d\n", __FILE__, __LINE__ );
|
||||
} else if ( rows == 0 ) {
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
continue;
|
||||
}
|
||||
|
||||
|
@ -224,7 +217,7 @@ AI_db_alertparser_thread ( void *arg )
|
|||
}
|
||||
}
|
||||
|
||||
lock_flag = false;
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
DB_free_result ( res );
|
||||
latest_time = time ( NULL );
|
||||
}
|
||||
|
@ -272,8 +265,13 @@ _AI_db_copy_alerts ( AI_snort_alert *node )
|
|||
AI_snort_alert*
|
||||
AI_db_get_alerts ()
|
||||
{
|
||||
while ( lock_flag );
|
||||
return _AI_db_copy_alerts ( alerts );
|
||||
AI_snort_alert *alerts_copy;
|
||||
|
||||
pthread_mutex_lock ( &mutex );
|
||||
alerts_copy = _AI_db_copy_alerts ( alerts );
|
||||
pthread_mutex_unlock ( &mutex );
|
||||
|
||||
return alerts_copy;
|
||||
} /* ----- end of function AI_db_get_alerts ----- */
|
||||
|
||||
/** @} */
|
||||
|
|
51
spp_ai.c
51
spp_ai.c
|
@ -135,10 +135,11 @@ static AI_config * AI_parse(char *args)
|
|||
{
|
||||
char *arg;
|
||||
char *match;
|
||||
char alertfile[1024] = { 0 };
|
||||
char clusterfile[1024] = { 0 };
|
||||
char corr_rules_dir[1024] = { 0 };
|
||||
char corr_alerts_dir[1024] = { 0 };
|
||||
char alertfile[1024] = { 0 };
|
||||
char clusterfile[1024] = { 0 };
|
||||
char corr_rules_dir[1024] = { 0 };
|
||||
char corr_alerts_dir[1024] = { 0 };
|
||||
char alert_history_file[1024] = { 0 };
|
||||
|
||||
char **matches = NULL;
|
||||
int nmatches = 0;
|
||||
|
@ -160,6 +161,7 @@ static AI_config * AI_parse(char *args)
|
|||
unsigned long cleanup_interval = 0,
|
||||
stream_expire_interval = 0,
|
||||
alertfile_len = 0,
|
||||
alert_history_file_len = 0,
|
||||
clusterfile_len = 0,
|
||||
corr_rules_dir_len = 0,
|
||||
corr_alerts_dir_len = 0,
|
||||
|
@ -176,7 +178,8 @@ static AI_config * AI_parse(char *args)
|
|||
has_clusterfile = false,
|
||||
has_corr_rules_dir = false,
|
||||
has_clustering = false,
|
||||
has_database_log = false;
|
||||
has_database_log = false,
|
||||
has_alert_history_file = false;
|
||||
|
||||
AI_config *config = NULL;
|
||||
|
||||
|
@ -336,6 +339,38 @@ static AI_config * AI_parse(char *args)
|
|||
}
|
||||
}
|
||||
|
||||
/* Parsing the alert_history_file option */
|
||||
if (( arg = (char*) strcasestr( args, "alert_history_file" ) ))
|
||||
{
|
||||
for ( arg += strlen("alert_history_file");
|
||||
*arg && *arg != '"';
|
||||
arg++ );
|
||||
|
||||
if ( !(*(arg++)) )
|
||||
{
|
||||
_dpd.fatalMsg("AIPreproc: alert_history_file option used but no filename specified\n");
|
||||
}
|
||||
|
||||
for ( alert_history_file[ (++alert_history_file_len)-1 ] = *arg;
|
||||
*arg && *arg != '"' && alert_history_file_len < 1024;
|
||||
arg++, alert_history_file[ (++alert_history_file_len)-1 ] = *arg );
|
||||
|
||||
if ( alert_history_file[0] == 0 || alert_history_file_len <= 1 ) {
|
||||
has_alert_history_file = false;
|
||||
} else {
|
||||
if ( alert_history_file_len >= 1024 ) {
|
||||
_dpd.fatalMsg("AIPreproc: alert_history_file path too long ( >= 1024 )\n");
|
||||
} else if ( strlen( alert_history_file ) == 0 ) {
|
||||
has_alert_history_file = false;
|
||||
} else {
|
||||
has_alert_history_file = true;
|
||||
alert_history_file [ alert_history_file_len-1 ] = 0;
|
||||
strncpy ( config->alert_history_file, alert_history_file, alert_history_file_len );
|
||||
_dpd.logMsg(" alert_history_file path: %s\n", config->alert_history_file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Parsing the clusterfile option */
|
||||
if (( arg = (char*) strcasestr( args, "clusterfile" ) ))
|
||||
{
|
||||
|
@ -767,6 +802,12 @@ static AI_config * AI_parse(char *args)
|
|||
alertparser_thread = AI_file_alertparser_thread;
|
||||
}
|
||||
|
||||
if ( !has_alert_history_file )
|
||||
{
|
||||
strncpy ( config->alert_history_file, DEFAULT_ALERT_HISTORY_FILE, sizeof ( config->alert_history_file ));
|
||||
has_alert_history_file = true;
|
||||
}
|
||||
|
||||
if ( has_clustering )
|
||||
{
|
||||
if ( ! hierarchy_nodes )
|
||||
|
|
9
spp_ai.h
9
spp_ai.h
|
@ -57,6 +57,9 @@
|
|||
/** Default directory for placing correlated alerts information (.dot and possibly .png files) */
|
||||
#define DEFAULT_CORR_ALERTS_DIR "/var/log/snort/correlated_alerts"
|
||||
|
||||
/** Default path to alert history binary file, used for bayesian statistical correlation over alerts */
|
||||
#define DEFAULT_ALERT_HISTORY_FILE "/var/log/snort/alert_history"
|
||||
|
||||
/** Default correlation threshold coefficient for correlating two hyperalerts */
|
||||
#define DEFAULT_CORR_THRESHOLD 0.5
|
||||
|
||||
|
@ -144,6 +147,9 @@ typedef struct
|
|||
/** Alert file */
|
||||
char alertfile[1024];
|
||||
|
||||
/** Alert history binary file */
|
||||
char alert_history_file[1024];
|
||||
|
||||
/** Clustered alerts file */
|
||||
char clusterfile[1024];
|
||||
|
||||
|
@ -305,6 +311,9 @@ struct pkt_info* AI_get_stream_by_key ( struct pkt_key );
|
|||
AI_snort_alert* AI_get_alerts ( void );
|
||||
AI_snort_alert* AI_get_clustered_alerts ( void );
|
||||
|
||||
void AI_serialize_alert ( AI_snort_alert*, AI_config* );
|
||||
void AI_deserialize_alert ( AI_snort_alert*, AI_config* );
|
||||
|
||||
/** Function pointer to the function used for getting the alert list (from log file, db, ...) */
|
||||
extern AI_snort_alert* (*get_alerts)(void);
|
||||
|
||||
|
|
Loading…
Reference in a new issue