blacklight
/
Snort_AIPreproc
mirror of https://github.com/BlackLight/Snort_AIPreproc.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improved multithread locks management

This commit is contained in:
BlackLight 2010-09-20 14:39:08 +02:00
parent 960b70e106
commit 93e0ba6511
9 changed files with 140 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
README
View File

@ -147,7 +147,8 @@ following:
preprocessor ai: \
hashtable_cleanup_interval 300 \
tcp_stream_expire_interval 300 \
alertfile "/home/youruser/local/snort/log/alert" \
alertfile "/your/snort/dir/log/alert" \
alert_history_file "/your/snort/dir/log/alert_history" \
alert_clustering_interval 300 \
correlation_graph_interval 300 \
correlation_rules_dir "/your/snort/dir/etc/corr_rules" \
@ -178,6 +179,10 @@ stream as "expired", if no more packets are received inside of that and it's not
- alertfile: The file where Snort saves its alerts, if they are saved to a file
and not to a database (default if not specified: /var/log/snort/alert)
- alert_history_file: The file keeping track of the history, in binary format,
of all the alerts received by the IDS, so that the module can build some
statistical correlation inferences over the past
- alert_clustering_interval: The interval that should occur from the clustering
of the alerts in the log according to the provided clustering hierarchies and
the next one (default if not specified: 300 seconds)

1
TODO
View File

@ -2,7 +2,6 @@
AVERAGE/HIGH PRIORITY:
======================
- Dynamic k parameter in correlation threshold
- Testing more scenarios, making more hyperalert models
- Bayesian learning among alerts in alert log
- libgc support

21
alert_history.c Normal file
View File

@ -0,0 +1,21 @@
/*
* =====================================================================================
*
* Filename: alert_history.c
*
* Description: Manages the history of alerts on a binary file
*
* Version: 0.1
* Created: 18/09/2010 21:02:15
* Revision: none
* Compiler: gcc
*
* Author: BlackLight (http://0x00.ath.cx), <blacklight@autistici.org>
* Licence: GNU GPL v.3
* Company: DO WHAT YOU WANT CAUSE A PIRATE IS FREE, YOU ARE A PIRATE!
*
* =====================================================================================
*/
#include "spp_ai.h"

26
alert_parser.c
View File

@ -29,9 +29,9 @@
#include <pthread.h>
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE FILE *alert_fp = NULL;
PRIVATE BOOL lock_flag = false;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE FILE *alert_fp = NULL;
PRIVATE pthread_mutex_t mutex;
/** \defgroup alert_parser Parse the alert log into binary structures
* @{ */
@ -78,6 +78,8 @@ AI_file_alertparser_thread ( void* arg )
AI_snort_alert *tmp = NULL;
BOOL in_alert = false;
pthread_mutex_init ( &mutex, NULL );
while ( 1 )
{
#ifndef MACOS
@ -132,6 +134,7 @@ AI_file_alertparser_thread ( void* arg )
fd = fileno(alert_fp);
}
}
/*
* Cause the thread to wait until a new file modification (a new alert).
*/
@ -139,6 +142,7 @@ AI_file_alertparser_thread ( void* arg )
usleep(100);
fstats( fd, &stats );
}
/*
* The first time the thread is called, the flow exits instantly from the while,
* so this first time the stats structure has to be initialized properly.
@ -153,7 +157,6 @@ AI_file_alertparser_thread ( void* arg )
#endif
/* Set the lock flag to true until it's done with alert parsing */
lock_flag = true;
while ( !feof ( alert_fp ))
{
@ -205,6 +208,8 @@ AI_file_alertparser_thread ( void* arg )
if ( !in_alert )
{
pthread_mutex_lock ( &mutex );
if ( preg_match ( "^\\[\\*\\*\\]\\s*\\[([0-9]+):([0-9]+):([0-9]+)\\]\\s*(.*)\\s*\\[\\*\\*\\]$", line, &matches, &nmatches ) > 0 )
{
in_alert = true;
@ -351,9 +356,11 @@ AI_file_alertparser_thread ( void* arg )
}
}
lock_flag = false;
pthread_mutex_unlock ( &mutex );
/* AI_alert_serialize ( alert, conf ); */
}
pthread_mutex_destroy ( &mutex );
pthread_exit ((void*) 0 );
return (void*) 0;
} /* ----- end of function AI_file_alertparser_thread ----- */
@ -397,8 +404,13 @@ _AI_copy_alerts ( AI_snort_alert *node )
AI_snort_alert*
AI_get_alerts ()
{
while ( lock_flag );
return _AI_copy_alerts ( alerts );
AI_snort_alert *alerts_copy;
pthread_mutex_lock ( &mutex );
alerts_copy = _AI_copy_alerts ( alerts );
pthread_mutex_unlock ( &mutex );
return alerts_copy;
} /* ----- end of function AI_get_alerts ----- */

33
cluster.c
View File

@ -50,11 +50,10 @@ typedef struct {
} AI_alert_occurrence;
PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL };
PRIVATE AI_config *_config = NULL;
PRIVATE AI_snort_alert *alert_log = NULL;
PRIVATE BOOL lock_flag = false;
PRIVATE hierarchy_node *h_root[CLUSTER_TYPES] = { NULL };
PRIVATE AI_config *_config = NULL;
PRIVATE AI_snort_alert *alert_log = NULL;
PRIVATE pthread_mutex_t mutex;
/**
* \brief Function that picks up the heuristic value for a clustering attribute in according to Julisch's heuristic (ACM, Vol.2, No.3, 09 2002, pag.124)
@ -373,7 +372,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
{
AI_snort_alert *tmp;
char ip[INET_ADDRSTRLEN];
char *timestamp;
char timestamp[128];
struct tm *_tm;
for ( tmp = log; tmp; tmp = tmp->next )
{
@ -384,8 +384,8 @@ _AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
fprintf ( fp, "[Priority: %d]\n", tmp->priority );
timestamp = ctime ( &tmp->timestamp );
timestamp[ strlen(timestamp)-1 ] = 0;
_tm = localtime ( &tmp->timestamp );
strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm );
fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
if ( h_root[src_addr] && tmp->h_node[src_addr] )
@ -445,13 +445,15 @@ _AI_cluster_thread ( void* arg )
int single_alerts_count = 0;
double heterogeneity = 0;
pthread_mutex_init ( &mutex, NULL );
while ( 1 )
{
/* Between an execution of the thread and the next one, sleep for alert_clustering_interval seconds */
sleep ( _config->alertClusteringInterval );
/* Set the lock over the alert log until it's done with the clustering operation */
lock_flag = true;
pthread_mutex_lock ( &mutex );
/* Free the current alert log and get the latest one */
if ( alert_log )
@ -462,7 +464,7 @@ _AI_cluster_thread ( void* arg )
if ( !( alert_log = get_alerts() ))
{
lock_flag = false;
pthread_mutex_unlock ( &mutex );
continue;
}
@ -561,7 +563,7 @@ _AI_cluster_thread ( void* arg )
alert_count -= _AI_merge_alerts ( &alert_log );
} while ( old_alert_count != alert_count );
lock_flag = false;
pthread_mutex_unlock ( &mutex );
if ( !( cluster_fp = fopen ( _config->clusterfile, "w" )) )
{
@ -732,8 +734,13 @@ _AI_copy_clustered_alerts ( AI_snort_alert *node )
AI_snort_alert*
AI_get_clustered_alerts ()
{
for ( ; lock_flag; usleep(100) );
return _AI_copy_clustered_alerts ( alert_log );
AI_snort_alert *alerts_copy;
pthread_mutex_lock ( &mutex );
alerts_copy = _AI_copy_clustered_alerts ( alert_log );
pthread_mutex_unlock ( &mutex );
return alerts_copy;
} /* ----- end of function AI_get_clustered_alerts ----- */
/** @} */

9
correlation.c
View File

@ -70,7 +70,7 @@ PRIVATE AI_hyperalert_info *hyperalerts = NULL;
PRIVATE AI_config *conf = NULL;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE AI_alert_correlation *correlation_table = NULL;
PRIVATE BOOL lock_flag = false;
PRIVATE pthread_mutex_t mutex;
/**
* \brief Clean up the correlation hash table
@ -713,6 +713,7 @@ AI_alert_correlation_thread ( void *arg )
#endif
conf = (AI_config*) arg;
pthread_mutex_init ( &mutex, NULL );
while ( 1 )
{
@ -727,7 +728,7 @@ AI_alert_correlation_thread ( void *arg )
}
/* Set the lock flag to true, and keep it this way until I've done with generating the new hyperalerts */
lock_flag = true;
pthread_mutex_lock ( &mutex );
if ( alerts )
{
@ -737,7 +738,7 @@ AI_alert_correlation_thread ( void *arg )
if ( !( alerts = AI_get_clustered_alerts() ))
{
lock_flag = false;
pthread_mutex_unlock ( &mutex );
continue;
}
@ -894,7 +895,7 @@ AI_alert_correlation_thread ( void *arg )
#endif
}
lock_flag = false;
pthread_mutex_unlock ( &mutex );
}
pthread_exit (( void* ) 0 );

30
db.c
View File

@ -30,12 +30,9 @@
* @{ */
PRIVATE AI_config *config;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE BOOL lock_flag = false;
/** pthread mutex for accessing database data */
PRIVATE pthread_mutex_t db_mutex = PTHREAD_MUTEX_INITIALIZER;
PRIVATE AI_config *config;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE pthread_mutex_t mutex;
/**
* \brief Thread for parsing alerts from a database
@ -65,7 +62,7 @@ AI_db_alertparser_thread ( void *arg )
}
config = ( AI_config* ) arg;
pthread_mutex_lock ( &db_mutex );
pthread_mutex_init ( &mutex, NULL );
if ( !DB_init ( config ))
{
@ -73,14 +70,10 @@ AI_db_alertparser_thread ( void *arg )
config->dbname, config->dbhost );
}
pthread_mutex_unlock ( &db_mutex );
while ( 1 )
{
sleep ( config->databaseParsingInterval );
/* Set the lock flag to true until it's done with alert parsing */
lock_flag = true;
pthread_mutex_lock ( &mutex );
memset ( query, 0, sizeof ( query ));
snprintf ( query, sizeof (query), "select cid, unix_timestamp(timestamp), signature from event where cid > %d "
@ -98,7 +91,7 @@ AI_db_alertparser_thread ( void *arg )
DB_close();
_dpd.fatalMsg ( "AIPreproc: Could not store the query result at %s:%d\n", __FILE__, __LINE__ );
} else if ( rows == 0 ) {
lock_flag = false;
pthread_mutex_unlock ( &mutex );
continue;
}
@ -224,7 +217,7 @@ AI_db_alertparser_thread ( void *arg )
}
}
lock_flag = false;
pthread_mutex_unlock ( &mutex );
DB_free_result ( res );
latest_time = time ( NULL );
}
@ -272,8 +265,13 @@ _AI_db_copy_alerts ( AI_snort_alert *node )
AI_snort_alert*
AI_db_get_alerts ()
{
while ( lock_flag );
return _AI_db_copy_alerts ( alerts );
AI_snort_alert *alerts_copy;
pthread_mutex_lock ( &mutex );
alerts_copy = _AI_db_copy_alerts ( alerts );
pthread_mutex_unlock ( &mutex );
return alerts_copy;
} /* ----- end of function AI_db_get_alerts ----- */
/** @} */

51
spp_ai.c
View File

@ -135,10 +135,11 @@ static AI_config * AI_parse(char *args)
{
char *arg;
char *match;
char alertfile[1024] = { 0 };
char clusterfile[1024] = { 0 };
char corr_rules_dir[1024] = { 0 };
char corr_alerts_dir[1024] = { 0 };
char alertfile[1024] = { 0 };
char clusterfile[1024] = { 0 };
char corr_rules_dir[1024] = { 0 };
char corr_alerts_dir[1024] = { 0 };
char alert_history_file[1024] = { 0 };
char **matches = NULL;
int nmatches = 0;
@ -160,6 +161,7 @@ static AI_config * AI_parse(char *args)
unsigned long cleanup_interval = 0,
stream_expire_interval = 0,
alertfile_len = 0,
alert_history_file_len = 0,
clusterfile_len = 0,
corr_rules_dir_len = 0,
corr_alerts_dir_len = 0,
@ -176,7 +178,8 @@ static AI_config * AI_parse(char *args)
has_clusterfile = false,
has_corr_rules_dir = false,
has_clustering = false,
has_database_log = false;
has_database_log = false,
has_alert_history_file = false;
AI_config *config = NULL;
@ -336,6 +339,38 @@ static AI_config * AI_parse(char *args)
}
}
/* Parsing the alert_history_file option */
if (( arg = (char*) strcasestr( args, "alert_history_file" ) ))
{
for ( arg += strlen("alert_history_file");
*arg && *arg != '"';
arg++ );
if ( !(*(arg++)) )
{
_dpd.fatalMsg("AIPreproc: alert_history_file option used but no filename specified\n");
}
for ( alert_history_file[ (++alert_history_file_len)-1 ] = *arg;
*arg && *arg != '"' && alert_history_file_len < 1024;
arg++, alert_history_file[ (++alert_history_file_len)-1 ] = *arg );
if ( alert_history_file[0] == 0 || alert_history_file_len <= 1 ) {
has_alert_history_file = false;
} else {
if ( alert_history_file_len >= 1024 ) {
_dpd.fatalMsg("AIPreproc: alert_history_file path too long ( >= 1024 )\n");
} else if ( strlen( alert_history_file ) == 0 ) {
has_alert_history_file = false;
} else {
has_alert_history_file = true;
alert_history_file [ alert_history_file_len-1 ] = 0;
strncpy ( config->alert_history_file, alert_history_file, alert_history_file_len );
_dpd.logMsg(" alert_history_file path: %s\n", config->alert_history_file);
}
}
}
/* Parsing the clusterfile option */
if (( arg = (char*) strcasestr( args, "clusterfile" ) ))
{
@ -767,6 +802,12 @@ static AI_config * AI_parse(char *args)
alertparser_thread = AI_file_alertparser_thread;
}
if ( !has_alert_history_file )
{
strncpy ( config->alert_history_file, DEFAULT_ALERT_HISTORY_FILE, sizeof ( config->alert_history_file ));
has_alert_history_file = true;
}
if ( has_clustering )
{
if ( ! hierarchy_nodes )

9
spp_ai.h
View File

@ -57,6 +57,9 @@
/** Default directory for placing correlated alerts information (.dot and possibly .png files) */
#define DEFAULT_CORR_ALERTS_DIR "/var/log/snort/correlated_alerts"
/** Default path to alert history binary file, used for bayesian statistical correlation over alerts */
#define DEFAULT_ALERT_HISTORY_FILE "/var/log/snort/alert_history"
/** Default correlation threshold coefficient for correlating two hyperalerts */
#define DEFAULT_CORR_THRESHOLD 0.5
@ -144,6 +147,9 @@ typedef struct
/** Alert file */
char alertfile[1024];
/** Alert history binary file */
char alert_history_file[1024];
/** Clustered alerts file */
char clusterfile[1024];
@ -305,6 +311,9 @@ struct pkt_info* AI_get_stream_by_key ( struct pkt_key );
AI_snort_alert* AI_get_alerts ( void );
AI_snort_alert* AI_get_clustered_alerts ( void );
void AI_serialize_alert ( AI_snort_alert*, AI_config* );
void AI_deserialize_alert ( AI_snort_alert*, AI_config* );
/** Function pointer to the function used for getting the alert list (from log file, db, ...) */
extern AI_snort_alert* (*get_alerts)(void);