Supporting manual (un)correlations from web interface

This commit is contained in:
BlackLight 2010-10-14 02:45:31 +02:00
parent e9dd3ebfa2
commit 544daa31cc
20 changed files with 904 additions and 73 deletions

View file

@ -56,7 +56,11 @@ fi
mkdir -p "${SHARE_PREFIX}/htdocs/js"
mkdir -p "${SHARE_PREFIX}/schemas"
install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_uncorrelations.xml" "${SHARE_PREFIX}/htdocs"
install -m 0755 "${PWD}/htdocs/pcap.cgi" "${SHARE_PREFIX}/htdocs"
install -m 0755 "${PWD}/htdocs/correlate.cgi" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/js/Curry-1.0.1.js" "${SHARE_PREFIX}/htdocs/js"
install -m 0644 "${PWD}/htdocs/js/dracula_algorithms.js" "${SHARE_PREFIX}/htdocs/js"
install -m 0644 "${PWD}/htdocs/js/dracula_graffle.js" "${SHARE_PREFIX}/htdocs/js"

View file

@ -832,7 +832,11 @@ fi
mkdir -p "${SHARE_PREFIX}/htdocs/js"
mkdir -p "${SHARE_PREFIX}/schemas"
install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/manual_uncorrelations.xml" "${SHARE_PREFIX}/htdocs"
install -m 0755 "${PWD}/htdocs/pcap.cgi" "${SHARE_PREFIX}/htdocs"
install -m 0755 "${PWD}/htdocs/correlate.cgi" "${SHARE_PREFIX}/htdocs"
install -m 0644 "${PWD}/htdocs/js/Curry-1.0.1.js" "${SHARE_PREFIX}/htdocs/js"
install -m 0644 "${PWD}/htdocs/js/dracula_algorithms.js" "${SHARE_PREFIX}/htdocs/js"
install -m 0644 "${PWD}/htdocs/js/dracula_graffle.js" "${SHARE_PREFIX}/htdocs/js"

3
TODO
View file

@ -2,13 +2,14 @@
AVERAGE/HIGH PRIORITY:
======================
- XML::Simple dependancy
- Manual alert correlation from the web interface
- Bayesian network
- Modules for correlation coefficients
- Code profiling
- Comment all the code!!!
- Neural network for computing k
- Testing more scenarios, making more hyperalert models
- Manual alert correlation from the web interface
=============
LOW PRIORITY:

View file

@ -351,16 +351,16 @@ AI_file_alertparser_thread ( void* arg )
_tm = localtime ( &stamp );
ltime.year = (unsigned short) _tm->tm_year + 1900;
ltime.day = (unsigned short) strtoul ( matches[0], NULL, 10 );
ltime.month = (unsigned short) strtoul ( matches[1], NULL, 10 );
ltime.day = (unsigned short) strtoul ( matches[1], NULL, 10 );
ltime.month = (unsigned short) strtoul ( matches[0], NULL, 10 );
ltime.hour = (unsigned short) strtoul ( matches[2], NULL, 10 );
ltime.min = (unsigned short) strtoul ( matches[3], NULL, 10 );
ltime.sec = (unsigned short) strtoul ( matches[4], NULL, 10 );
snprintf ( strtime, sizeof(strtime), "%02hu/%02hu/%04hu, %02hu:%02hu:%02hu",
ltime.day, ltime.month, ltime.year, ltime.hour, ltime.min, ltime.sec );
ltime.month, ltime.day, ltime.year, ltime.hour, ltime.min, ltime.sec );
strptime ( strtime, "%d/%m/%Y, %H:%M:%S", _tm );
strptime ( strtime, "%m/%d/%Y, %H:%M:%S", _tm );
alert->timestamp = mktime ( _tm );
alert->ip_src_addr = inet_addr ( matches[5] );
@ -380,16 +380,16 @@ AI_file_alertparser_thread ( void* arg )
_tm = localtime ( &stamp );
ltime.year = (unsigned short) _tm->tm_year + 1900;
ltime.day = (unsigned short) strtoul ( matches[0], NULL, 10 );
ltime.month = (unsigned short) strtoul ( matches[1], NULL, 10 );
ltime.day = (unsigned short) strtoul ( matches[1], NULL, 10 );
ltime.month = (unsigned short) strtoul ( matches[0], NULL, 10 );
ltime.hour = (unsigned short) strtoul ( matches[2], NULL, 10 );
ltime.min = (unsigned short) strtoul ( matches[3], NULL, 10 );
ltime.sec = (unsigned short) strtoul ( matches[4], NULL, 10 );
snprintf ( strtime, sizeof(strtime), "%02hu/%02hu/%04hu, %02hu:%02hu:%02hu",
ltime.day, ltime.month, ltime.year, ltime.hour, ltime.min, ltime.sec );
ltime.month, ltime.day, ltime.year, ltime.hour, ltime.min, ltime.sec );
strptime ( strtime, "%d/%m/%Y, %H:%M:%S", _tm );
strptime ( strtime, "%m/%d/%Y, %H:%M:%S", _tm );
alert->timestamp = mktime ( _tm );
alert->ip_src_addr = inet_addr ( matches[5] );

View file

@ -392,8 +392,7 @@ __AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
{
AI_snort_alert *tmp;
char ip[INET_ADDRSTRLEN];
char timestamp[128];
struct tm *_tm;
char *timestamp;
for ( tmp = log; tmp; tmp = tmp->next )
{
@ -404,8 +403,8 @@ __AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp )
fprintf ( fp, "[Priority: %d]\n", tmp->priority );
_tm = localtime ( &tmp->timestamp );
strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm );
timestamp = ctime ( &( tmp->timestamp ));
timestamp [ strlen ( timestamp ) - 1 ] = 0;
fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp );
if ( h_root[src_addr] && tmp->h_node[src_addr] )

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklighth//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<hyperalert>
<snort-id>1.1394.12</snort-id>

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklighth//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<hyperalert>
<snort-id>1.1924.8</snort-id>

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklighth//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<hyperalert>
<snort-id>1.469.4</snort-id>

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklighth//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<hyperalert>
<snort-id>1.579.10</snort-id>

View file

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklighth//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD HYPERALERT SNORT MODEL//EN" "http://0x00.ath.cx/hyperalert.dtd">
<hyperalert>
<snort-id>122.1.0</snort-id>

View file

@ -1,7 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert[
<!ELEMENT hyperalert (snort-id, pre, post)>
<!ELEMENT hyperalert (snort-id, pre, post, desc)>
<!ELEMENT snort-id (#PCDATA)>
<!ELEMENT desc (#PCDATA)>
<!ELEMENT pre (#PCDATA)>
<!ELEMENT post (#PCDATA)>
]>

View file

@ -41,12 +41,32 @@
#error "libxml2 reader not enabled\n"
#endif
/** Enumeration for the types of XML tags */
enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, TAG_NUM };
/** Enumeration for the types of hyperalert XML tags */
enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, HYP_TAG_NUM };
/** Enumeration for the types of manual correlations XML tags */
enum { inCorrelation, inCorrelations, inFromTag, inToTag, MAN_TAG_NUM };
typedef struct {
int from_gid;
int from_sid;
int from_rev;
int to_gid;
int to_sid;
int to_rev;
} AI_alert_type_pair_key;
typedef struct {
AI_alert_type_pair_key key;
enum { manuallyNone, manuallyCorrelated, manuallyNotCorrelated } corr_type;
UT_hash_handle hh;
} AI_alert_type_pair;
PRIVATE AI_hyperalert_info *hyperalerts = NULL;
PRIVATE AI_snort_alert *alerts = NULL;
PRIVATE AI_alert_correlation *correlation_table = NULL;
PRIVATE AI_alert_type_pair *manual_correlations = NULL;
PRIVATE AI_alert_type_pair *manual_uncorrelations = NULL;
PRIVATE pthread_mutex_t mutex;
/**
@ -83,10 +103,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp )
dst_port1[10],
src_port2[10],
dst_port2[10],
timestamp1[40],
timestamp2[40];
struct tm *t1, *t2;
*timestamp1,
*timestamp2;
if ( !corr )
return;
@ -97,8 +115,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp )
snprintf ( src_port1, sizeof ( src_port1 ), "%d", ntohs ( corr->key.a->tcp_src_port ));
snprintf ( dst_port1, sizeof ( dst_port1 ), "%d", ntohs ( corr->key.a->tcp_dst_port ));
t1 = localtime ( &(corr->key.a->timestamp ));
strftime ( timestamp1, sizeof ( timestamp1 ), "%a %b %d %Y, %H:%M:%S", t1 );
timestamp1 = ctime ( &(corr->key.a->timestamp ) );
timestamp1 [ strlen ( timestamp1 ) - 1 ] = 0;
inet_ntop ( AF_INET, &(corr->key.b->ip_src_addr), src_addr2, INET_ADDRSTRLEN );
inet_ntop ( AF_INET, &(corr->key.b->ip_dst_addr), dst_addr2, INET_ADDRSTRLEN );
@ -106,8 +124,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp )
snprintf ( src_port2, sizeof ( src_port2 ), "%d", ntohs ( corr->key.b->tcp_src_port ));
snprintf ( dst_port2, sizeof ( dst_port2 ), "%d", ntohs ( corr->key.b->tcp_dst_port ));
t2 = localtime ( &(corr->key.b->timestamp ));
strftime ( timestamp2, sizeof ( timestamp2 ), "%a %b %d %Y, %H:%M:%S", t2 );
timestamp2 = ctime ( &(corr->key.b->timestamp ) );
timestamp2 [ strlen ( timestamp2 ) - 1 ] = 0;
fprintf ( fp,
"\t\"[%d.%d.%d] %s\\n"
@ -178,12 +196,18 @@ __AI_correlated_alerts_to_json ()
fprintf ( fp, "{\n"
"\t\"id\": %lu,\n"
"\t\"snortSID\": \"%u\",\n"
"\t\"snortGID\": \"%u\",\n"
"\t\"snortREV\": \"%u\",\n"
"\t\"label\": \"%s\",\n"
"\t\"date\": \"%s\",\n"
"\t\"clusteredAlertsCount\": %u,\n"
"\t\"from\": \"%s:%s\",\n"
"\t\"to\": \"%s:%s\"",
alert_iterator->alert_id,
alert_iterator->sid,
alert_iterator->gid,
alert_iterator->rev,
alert_iterator->desc,
strtime,
alert_iterator->grouped_alerts_count,
@ -617,6 +641,348 @@ __AI_kb_correlation_coefficient ( AI_snort_alert *a, AI_snort_alert *b )
} /* ----- end of function __AI_kb_correlation_coefficient ----- */
/**
* \brief Parse the manual specified correlations from XML file(s) and fills the hash table
*/
PRIVATE void*
__AI_manual_correlations_parsing_thread ( void *arg )
{
unsigned int i = 0;
char manual_correlations_xml[1060] = { 0 },
manual_uncorrelations_xml[1060] = { 0 };
struct stat st;
xmlTextReaderPtr xml;
const xmlChar *tagname;
AI_alert_type_pair_key key;
AI_alert_type_pair *pair = NULL,
*found = NULL;
BOOL xml_flags[MAN_TAG_NUM] = { false };
while ( 1 )
{
/* Cleanup tables */
while ( manual_correlations )
{
pair = manual_correlations;
HASH_DEL ( manual_correlations, pair );
free ( pair );
}
while ( manual_uncorrelations )
{
pair = manual_uncorrelations;
HASH_DEL ( manual_uncorrelations, pair );
free ( pair );
}
pair = NULL;
memset ( &key, 0, sizeof ( key ));
snprintf ( manual_correlations_xml,
sizeof ( manual_correlations_xml ),
"%s/manual_correlations.xml", config->webserv_dir );
snprintf ( manual_uncorrelations_xml,
sizeof ( manual_uncorrelations_xml ),
"%s/manual_uncorrelations.xml", config->webserv_dir );
if ( stat ( manual_correlations_xml, &st ) < 0 )
{
pthread_exit ((void*) 0);
return (void*) 0;
}
if ( stat ( manual_uncorrelations_xml, &st ) < 0 )
{
pthread_exit ((void*) 0);
return (void*) 0;
}
LIBXML_TEST_VERSION
/* Check manual correlations */
if ( !( xml = xmlReaderForFile ( manual_correlations_xml, NULL, 0 )))
{
pthread_exit ((void*) 0);
return (void*) 0;
}
while ( xmlTextReaderRead ( xml ))
{
if ( !( tagname = xmlTextReaderConstName ( xml )))
continue;
if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT )
{
if ( !strcasecmp ((const char*) tagname, "correlations" ))
{
if ( xml_flags[inCorrelations] )
{
AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelations] = true;
}
} else if ( !strcasecmp ((const char*) tagname, "correlation" )) {
if ( xml_flags[inCorrelation] )
{
AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelation] = true;
}
} else if ( !strcasecmp ((const char*) tagname, "from" )) {
xml_flags[inFromTag] = true;
key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0;
key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0;
key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0;
/* If this is a new pair, allocate the memory */
if ( pair == NULL )
{
if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
pair->corr_type = manuallyCorrelated;
} else {
/* Otherwise, add the pair to the hash, if it's not already there */
pair->key = key;
HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found );
if ( !found )
{
HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair );
}
pair = NULL;
memset ( &key, 0, sizeof ( key ));
}
} else if ( !strcasecmp ((const char*) tagname, "to" )) {
xml_flags[inToTag] = true;
key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0;
key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0;
key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0;
/* If this is a new pair, allocate the memory */
if ( pair == NULL )
{
if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
pair->corr_type = manuallyCorrelated;
} else {
/* Otherwise, add the pair to the hash, if it's not already there */
pair->key = key;
HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found );
if ( !found )
{
HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair );
}
pair = NULL;
memset ( &key, 0, sizeof ( key ));
}
} else {
AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ );
}
} else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) {
if ( !strcasecmp ((const char*) tagname, "correlations" ))
{
if ( !xml_flags[inCorrelations] )
{
AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelations] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "correlation" )) {
if ( !xml_flags[inCorrelation] )
{
AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelation] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "from" )) {
if ( !xml_flags[inFromTag] )
{
AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inFromTag] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "to" )) {
if ( !xml_flags[inToTag] )
{
AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inToTag] = false;
}
} else {
AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ );
}
}
}
xmlFreeTextReader ( xml );
xmlCleanupParser();
for ( i=0; i < MAN_TAG_NUM; i++ )
{
xml_flags[i] = false;
}
/* Check manual un-correlations */
if ( !( xml = xmlReaderForFile ( manual_uncorrelations_xml, NULL, 0 )))
{
pthread_exit ((void*) 0);
return (void*) 0;
}
while ( xmlTextReaderRead ( xml ))
{
if ( !( tagname = xmlTextReaderConstName ( xml )))
continue;
if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT )
{
if ( !strcasecmp ((const char*) tagname, "correlations" ))
{
if ( xml_flags[inCorrelations] )
{
AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelations] = true;
}
} else if ( !strcasecmp ((const char*) tagname, "correlation" )) {
if ( xml_flags[inCorrelation] )
{
AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelation] = true;
}
} else if ( !strcasecmp ((const char*) tagname, "from" )) {
xml_flags[inFromTag] = true;
key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0;
key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0;
key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0;
/* If this is a new pair, allocate the memory */
if ( pair == NULL )
{
if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
pair->corr_type = manuallyNotCorrelated;
} else {
/* Otherwise, add the pair to the hash, if it's not already there */
pair->key = key;
HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found );
if ( !found )
{
HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair );
}
pair = NULL;
memset ( &key, 0, sizeof ( key ));
}
} else if ( !strcasecmp ((const char*) tagname, "to" )) {
xml_flags[inToTag] = true;
key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0;
key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0;
key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ?
strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0;
/* If this is a new pair, allocate the memory */
if ( pair == NULL )
{
if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair ))))
{
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
}
pair->corr_type = manuallyNotCorrelated;
} else {
/* Otherwise, add the pair to the hash, if it's not already there */
pair->key = key;
HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found );
if ( !found )
{
HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair );
}
pair = NULL;
memset ( &key, 0, sizeof ( key ));
}
} else {
AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ );
}
} else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) {
if ( !strcasecmp ((const char*) tagname, "correlations" ))
{
if ( !xml_flags[inCorrelations] )
{
AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelations] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "correlation" )) {
if ( !xml_flags[inCorrelation] )
{
AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inCorrelation] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "from" )) {
if ( !xml_flags[inFromTag] )
{
AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inFromTag] = false;
}
} else if ( !strcasecmp ((const char*) tagname, "to" )) {
if ( !xml_flags[inToTag] )
{
AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ );
} else {
xml_flags[inToTag] = false;
}
} else {
AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ );
}
}
}
xmlFreeTextReader ( xml );
xmlCleanupParser();
sleep ( config->manualCorrelationsParsingInterval );
}
pthread_exit ((void*) 0);
return (void*) 0;
} /* ----- end of function __AI_manual_correlations_parsing_thread ----- */
/**
* \brief Substitute the macros in hyperalert pre-conditions and post-conditions with their associated values
* \param alert Reference to the hyperalert to work on
@ -719,7 +1085,7 @@ __AI_hyperalert_from_XML ( AI_hyperalert_key key )
{
char hyperalert_file[1024] = {0};
char snort_id[1024] = {0};
BOOL xmlFlags[TAG_NUM] = { false };
BOOL xmlFlags[HYP_TAG_NUM] = { false };
struct stat st;
xmlTextReaderPtr xml;
const xmlChar *tagname, *tagvalue;
@ -867,13 +1233,18 @@ AI_alert_correlation_thread ( void *arg )
AI_alert_correlation_key corr_key;
AI_alert_correlation *corr = NULL;
AI_alert_type_pair_key pair_key;
AI_alert_type_pair *pair = NULL,
*unpair = NULL;
AI_hyperalert_key key;
AI_hyperalert_info *hyp = NULL;
AI_snort_alert *alert_iterator = NULL,
*alert_iterator2 = NULL;
pthread_t db_thread;
pthread_t db_thread,
manual_corr_thread;
#ifdef HAVE_LIBGVC
char corr_png_file[4096] = { 0 };
@ -883,6 +1254,12 @@ AI_alert_correlation_thread ( void *arg )
pthread_mutex_init ( &mutex, NULL );
/* Start the thread for parsing manual correlations from XML */
if ( pthread_create ( &manual_corr_thread, NULL, __AI_manual_correlations_parsing_thread, NULL ) != 0 )
{
AI_fatal_err ( "Failed to create the manual correlations parsing thread", __FILE__, __LINE__ );
}
while ( 1 )
{
sleep ( config->correlationGraphInterval );
@ -955,6 +1332,7 @@ AI_alert_correlation_thread ( void *arg )
__AI_correlation_table_cleanup();
correlation_table = NULL;
/* Fill the table of correlated alerts */
for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next )
{
for ( alert_iterator2 = alerts; alert_iterator2; alert_iterator2 = alert_iterator2->next )
@ -1026,12 +1404,23 @@ AI_alert_correlation_thread ( void *arg )
/* Find correlated alerts */
for ( corr = correlation_table; corr; corr = ( AI_alert_correlation* ) corr->hh.next )
{
if ( corr->correlation >= corr_threshold &&
pair_key.from_sid = corr->key.a->sid;
pair_key.from_gid = corr->key.a->gid;
pair_key.from_rev = corr->key.a->rev;
pair_key.to_sid = corr->key.b->sid;
pair_key.to_gid = corr->key.b->gid;
pair_key.to_rev = corr->key.b->rev;
HASH_FIND ( hh, manual_correlations, &pair_key, sizeof ( pair_key ), pair );
HASH_FIND ( hh, manual_uncorrelations, &pair_key, sizeof ( pair_key ), unpair );
if ( !unpair && ( pair || (
corr->correlation >= corr_threshold &&
corr_threshold != 0.0 &&
corr->key.a->timestamp <= corr->key.b->timestamp && ! (
corr->key.a->gid == corr->key.b->gid &&
corr->key.a->sid == corr->key.b->sid &&
corr->key.a->rev == corr->key.b->rev ))
corr->key.a->rev == corr->key.b->rev ))))
{
if ( !( corr->key.a->derived_alerts = ( AI_snort_alert** ) realloc ( corr->key.a->derived_alerts, (++corr->key.a->n_derived_alerts) * sizeof ( AI_snort_alert* ))))
AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );

178
htdocs/correlate.cgi Executable file
View file

@ -0,0 +1,178 @@
#!/usr/bin/env perl
use strict;
use warnings;
use XML::Simple;
use Data::Dumper;
use Env qw(QUERY_STRING DOCUMENT_ROOT);
exit 1 if ( !$QUERY_STRING );
$QUERY_STRING =~ /action=([a-z]+)/;
my $action = $1;
$QUERY_STRING =~ /from_sid=([0-9]+)/;
my $from_sid = $1;
$QUERY_STRING =~ /from_gid=([0-9]+)/;
my $from_gid = $1;
$QUERY_STRING =~ /from_rev=([0-9]+)/;
my $from_rev = $1;
$QUERY_STRING =~ /to_sid=([0-9]+)/;
my $to_sid = $1;
$QUERY_STRING =~ /to_gid=([0-9]+)/;
my $to_gid = $1;
$QUERY_STRING =~ /to_rev=([0-9]+)/;
my $to_rev = $1;
exit 1 unless (
defined ( $action ) &&
defined ( $from_sid ) &&
defined ( $from_gid ) &&
defined ( $from_rev ) &&
defined ( $to_sid ) &&
defined ( $to_gid ) &&
defined ( $to_rev ));
my $xml = new XML::Simple ( forcearray => 1 );
my $corr_data = $xml->XMLin ( "$DOCUMENT_ROOT/manual_correlations.xml" );
my $uncorr_data = $xml->XMLin ( "$DOCUMENT_ROOT/manual_uncorrelations.xml" );
if ( $action eq 'add' )
{
# Check if 'correlation' already contains this item
for my $node ( @{$corr_data->{'correlation'}} )
{
my $from = @{$node->{'from'}}[0];
my $to = @{$node->{'to'}}[0];
exit 1 if (
$from->{'sid'} eq $from_sid &&
$from->{'gid'} eq $from_gid &&
$from->{'rev'} eq $from_rev &&
$to->{'sid'} eq $to_sid &&
$to->{'gid'} eq $to_gid &&
$to->{'rev'} eq $to_rev
);
}
# If this node is in 'uncorrelated' alerts, remove it from there
if ( UNIVERSAL::isa ( $uncorr_data->{'correlation'}, "ARRAY" ))
{
for my $i ( 0..@{$uncorr_data->{'correlation'}}-1 )
{
if ( defined ( @{$uncorr_data->{'correlation'}}[$i] ))
{
my $from = @{@{$uncorr_data->{'correlation'}}[$i]->{'from'}}[0];
my $to = @{@{$uncorr_data->{'correlation'}}[$i]->{'to'}}[0];
splice ( @{$uncorr_data->{'correlation'}}, $i, 1 ) if (
$from->{'sid'} eq $from_sid &&
$from->{'gid'} eq $from_gid &&
$from->{'rev'} eq $from_rev &&
$to->{'sid'} eq $to_sid &&
$to->{'gid'} eq $to_gid &&
$to->{'rev'} eq $to_rev
);
}
}
}
my %hash = (
'to' => [
{
'sid' => $to_sid,
'gid' => $to_gid,
'rev' => $to_rev,
}
],
'from' => [
{
'sid' => $from_sid,
'gid' => $from_gid,
'rev' => $from_rev
}
]
);
push @{$corr_data->{'correlation'}}, \%hash;
} elsif ( $action eq 'remove' ) {
# Check if 'un-correlation' already contains this item
for my $node ( @{$uncorr_data->{'correlation'}} )
{
my $from = @{$node->{'from'}}[0];
my $to = @{$node->{'to'}}[0];
exit 1 if (
$from->{'sid'} eq $from_sid &&
$from->{'gid'} eq $from_gid &&
$from->{'rev'} eq $from_rev &&
$to->{'sid'} eq $to_sid &&
$to->{'gid'} eq $to_gid &&
$to->{'rev'} eq $to_rev
);
}
# If this node is in 'correlated' alerts, remove it from there
if ( UNIVERSAL::isa ( $corr_data->{'correlation'}, "ARRAY" ))
{
for my $i ( 0..@{$corr_data->{'correlation'}}-1 )
{
if ( defined ( @{$corr_data->{'correlation'}}[$i] ))
{
my $from = @{@{$corr_data->{'correlation'}}[$i]->{'from'}}[0];
my $to = @{@{$corr_data->{'correlation'}}[$i]->{'to'}}[0];
splice ( @{$corr_data->{'correlation'}}, $i, 1 ) if (
$from->{'sid'} eq $from_sid &&
$from->{'gid'} eq $from_gid &&
$from->{'rev'} eq $from_rev &&
$to->{'sid'} eq $to_sid &&
$to->{'gid'} eq $to_gid &&
$to->{'rev'} eq $to_rev
);
}
}
}
my %hash = (
'to' => [
{
'sid' => $to_sid,
'gid' => $to_gid,
'rev' => $to_rev
}
],
'from' => [
{
'sid' => $from_sid,
'gid' => $from_gid,
'rev' => $from_rev
}
]
);
push @{$uncorr_data->{'correlation'}}, \%hash;
}
my $xml_corr_out = $xml->XMLout ( $corr_data, RootName => "correlations" );
my $xml_uncorr_out = $xml->XMLout ( $uncorr_data, RootName => "correlations" );
open OUT, "> $DOCUMENT_ROOT/manual_correlations.xml" or exit 1;
print OUT "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
print OUT "<!DOCTYPE hyperalert PUBLIC \"-//blacklight//DTD MANUAL CORRELATIONS//EN\" \"http://0x00.ath.cx/manual_correlations.dtd\">\n\n";
print OUT $xml_corr_out."\n";
close OUT;
open OUT, "> $DOCUMENT_ROOT/manual_uncorrelations.xml" or exit 1;
print OUT "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
print OUT "<!DOCTYPE hyperalert PUBLIC \"-//blacklight//DTD MANUAL CORRELATIONS//EN\" \"http://0x00.ath.cx/manual_correlations.dtd\">\n\n";
print OUT $xml_uncorr_out."\n";
close OUT;
print "Content-Type: text/html\n\n";

View file

@ -1,19 +1,10 @@
<html>
<head>
<style rel="stylesheet" type="text/css">
#popupAlert {
z-index : 100;
display : none;
position : absolute;
border : 1px solid #000;
background-color : #89ffa7;
color : #000;
opacity : 0.6;
padding : 5px;
}
#alertInfo {
display : none;
font : Fixed;
font-size : 12px;
border : 1px solid #888;
padding : 3px;
height : 150px;
@ -26,7 +17,18 @@
}
a {
color : #22dd22;
color : #2d2;
}
button {
background-color : #ddd;
color : #222;
font-size : 13px;
border : 1px dotted #888;
}
button:hover {
background-color : #7fc;
}
</style>
@ -43,6 +45,60 @@ layouter = null;
renderer = null;
json = [];
inCorrelateFrom = false;
inCorrelateTo = false;
correlationFrom = -1;
correlationTo = -1;
correlationFromIndex = -1;
correlationToIndex = -1;
inUncorrelateFrom = false;
inUncorrelateTo = false;
uncorrelationFrom = -1;
uncorrelationTo = -1;
uncorrelationFromIndex = -1;
uncorrelationToIndex = -1;
function correlate() {
var div = document.getElementById ( 'alertInfo' );
var button = document.getElementById ( 'correlate' );
if ( inCorrelateFrom || inCorrelateTo )
{
inCorrelateFrom = false;
div.style.backgroundColor = '#FFF';
div.style.display = 'none';
button.innerHTML = 'Manually correlate';
} else {
inCorrelateFrom = true;
inCorrelateTo = false;
div.innerHTML = '<b>Click on the source alert of the correlation</b>';
div.style.backgroundColor = '#FF9';
div.style.display = 'block';
button.innerHTML = 'Cancel manual correlation';
}
}
function uncorrelate() {
var div = document.getElementById ( 'alertInfo' );
var button = document.getElementById ( 'uncorrelate' );
if ( inUncorrelateFrom || inUncorrelateTo )
{
inUncorrelateFrom = false;
div.style.backgroundColor = '#FFF';
div.style.display = 'none';
button.innerHTML = 'Remove correlation';
} else {
inUncorrelateFrom = true;
inUncorrelateTo = false;
div.innerHTML = '<b>Click on the source alert of the un-correlation</b>';
div.style.backgroundColor = '#FF9';
div.style.display = 'block';
button.innerHTML = 'Cancel un-correlation';
}
}
window.onload = function() {
var req = new XMLHttpRequest();
@ -116,17 +172,150 @@ window.onload = function() {
{
var id = this.id.replace ( /^alert/, "" );
var json_element = null;
var index = -1;
for ( var i=0; i < json.length; i++ )
{
if ( json[i].id == id )
{
json_element = json[i];
index = i;
break;
}
}
div.style.display = 'block';
// Correlation stuff
if ( inCorrelateFrom )
{
inCorrelateFrom = false;
inCorrelateTo = true;
correlationFrom = id;
correlationFromIndex = index;
div.innerHTML = "<b>Click on the destination alert of the correlation</b><br>\n" +
"From: <b>" + json_element.label + "</b>";
return;
} else if ( inCorrelateTo ) {
inCorrelateTo = false;
correlationTo = id;
correlationToIndex = index;
div.style.backgroundColor = '#FFF';
div.style.display = 'none';
// Check if this correlation is already there
for ( var j=0; j < connections.length; j++ )
{
if ( connections[j].from == correlationFrom &&
connections[j].to == correlationTo )
{
document.getElementById ( 'correlate' ).innerHTML = "Manually correlate";
return;
}
}
var from_sid = json[correlationFromIndex].snortSID;
var from_gid = json[correlationFromIndex].snortGID;
var from_rev = json[correlationFromIndex].snortREV;
var to_sid = json[correlationToIndex].snortSID;
var to_gid = json[correlationToIndex].snortGID;
var to_rev = json[correlationToIndex].snortREV;
var corr_req = new XMLHttpRequest();
corr_req.open ( 'GET', 'http://' + window.location.host +
'/correlate.cgi?' +
'action=add' +
'&from_sid=' + from_sid +
'&from_gid=' + from_gid +
'&from_rev=' + from_rev +
'&to_sid=' + to_sid +
'&to_gid=' + to_gid +
'&to_rev=' + to_rev, true );
corr_req.send ( null );
document.getElementById ( 'correlate' ).innerHTML = "Manually correlate";
connections.push ({
"from": correlationFrom,
"to" : correlationTo
});
g.addEdge ( correlationFrom, correlationTo, { directed: true });
redraw();
return;
}
// Un-correlation stuff
if ( inUncorrelateFrom )
{
inUncorrelateFrom = false;
inUncorrelateTo = true;
uncorrelationFrom = id;
uncorrelationFromIndex = index;
div.innerHTML = "<b>Click on the destination alert of the un-correlation</b><br>\n" +
"From: <b>" + json_element.label + "</b>";
return;
} else if ( inUncorrelateTo ) {
inUncorrelateTo = false;
uncorrelationTo = id;
uncorrelationToIndex = index;
div.style.backgroundColor = '#FFF';
div.style.display = 'none';
// Check if the alerts are already un-correlated
var correlated = false;
for ( var j=0; j < connections.length && !correlated; j++ )
{
if ( connections[j].from == uncorrelationFrom &&
connections[j].to == uncorrelationTo )
{
correlated = true;
}
}
if ( !correlated )
{
document.getElementById ( 'uncorrelate' ).innerHTML = "Remove correlation";
return;
}
var from_sid = json[uncorrelationFromIndex].snortSID;
var from_gid = json[uncorrelationFromIndex].snortGID;
var from_rev = json[uncorrelationFromIndex].snortREV;
var to_sid = json[uncorrelationToIndex].snortSID;
var to_gid = json[uncorrelationToIndex].snortGID;
var to_rev = json[uncorrelationToIndex].snortREV;
var corr_req = new XMLHttpRequest();
corr_req.open ( 'GET', 'http://' + window.location.host +
'/correlate.cgi?' +
'action=remove' +
'&from_sid=' + from_sid +
'&from_gid=' + from_gid +
'&from_rev=' + from_rev +
'&to_sid=' + to_sid +
'&to_gid=' + to_gid +
'&to_rev=' + to_rev, true );
corr_req.send ( null );
document.getElementById ( 'uncorrelate' ).innerHTML = "Remove correlation";
// Remove the connection from the array
for ( var j=0; j < connections.length; j++ )
{
if ( connections[j].from == uncorrelationFrom &&
connections[j].to == uncorrelationTo )
{
connections.slice ( j, 1 );
break;
}
}
redraw();
return;
}
var content =
'<table id="alertInfoTable" cellspacing="5"><tr style="background-color:#99ff99">' +
'<td><b>Type</b></td><td><b>From</b></td>' +
@ -143,7 +332,9 @@ window.onload = function() {
json_element.packets[j].match ( /^.*(=*)$/ );
var pad = RegExp.$1;
var len = parseInt ( json_element.packets[j].length * 3 / 4 ) - pad.length;
json_element.date.match ( /[a-zA-Z]+\s+([a-zA-Z]+)\s+([0-9]+)\s+([0-9]+):([0-9]+):([0-9]+)\s+([0-9]+)/ );
var month = RegExp.$1;
var day = RegExp.$2;
var hour = RegExp.$3;
@ -245,6 +436,7 @@ window.onload = function() {
}
content += '</table>';
div.style.display = 'block';
div.innerHTML = content;
}
};
@ -261,7 +453,10 @@ window.onload = function() {
</head>
<body>
<div id="canvas" style="border: 1px solid #000"></div>
<button id="redraw" onclick="redraw();">Redraw</button>
<div style="height : 5px"></div>
<button id="redraw" onClick="redraw();">Redraw</button>
<button id="correlate" onClick="correlate();">Manually correlate</button>
<button id="uncorrelate" onClick="uncorrelate();">Remove correlation</button>
<div id="popupAlert"></div>
<div id="alertInfo"></div>

View file

@ -0,0 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE correlations[
<!ELEMENT correlations (correlation)*>
<!ELEMENT correlation (from, to)>
<!ATTLIST from
sid CDATA #REQUIRED
gid CDATA #REQUIRED
rev CDATA #REQUIRED
>
<!ATTLIST to
sid CDATA #REQUIRED
gid CDATA #REQUIRED
rev CDATA #REQUIRED
>
]>

View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD MANUAL CORRELATIONS//EN" "http://0x00.ath.cx/manual_correlations.dtd">
<correlations>
</correlations>

View file

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE hyperalert PUBLIC "-//blacklight//DTD MANUAL CORRELATIONS//EN" "http://0x00.ath.cx/manual_correlations.dtd">
<correlations>
</correlations>

View file

@ -1,4 +1,4 @@
#!/usr/bin/perl
#!/usr/bin/env perl
use strict;
use warnings;

View file

@ -203,7 +203,8 @@ static AI_config * AI_parse(char *args)
webserv_banner_len = 0,
alert_clustering_interval = 0,
database_parsing_interval = 0,
correlation_graph_interval = 0;
correlation_graph_interval = 0,
manual_correlations_parsing_interval = 0;
BOOL has_cleanup_interval = false,
has_stream_expire_interval = false,
@ -420,6 +421,27 @@ static AI_config * AI_parse(char *args)
config->bayesianCorrelationInterval = bayesian_correlation_interval;
_dpd.logMsg( " Bayesian correlation interval: %u\n", config->bayesianCorrelationInterval );
/* Parsing the manual_correlations_parsing_interval option */
if (( arg = (char*) strcasestr( args, "manual_correlations_parsing_interval" ) ))
{
for ( arg += strlen("manual_correlations_parsing_interval");
*arg && (*arg < '0' || *arg > '9');
arg++ );
if ( !(*arg) )
{
AI_fatal_err ( "manual_correlations_parsing_interval option used but "
"no value specified", __FILE__, __LINE__ );
}
manual_correlations_parsing_interval = strtoul ( arg, NULL, 10 );
} else {
manual_correlations_parsing_interval = DEFAULT_MANUAL_CORRELATIONS_PARSING_INTERVAL;
}
config->manualCorrelationsParsingInterval = manual_correlations_parsing_interval;
_dpd.logMsg( " Manual correlations parsing interval: %u\n", config->manualCorrelationsParsingInterval );
/* Parsing the bayesian_correlation_cache_validity option */
if (( arg = (char*) strcasestr( args, "bayesian_correlation_cache_validity" ) ))
{

View file

@ -72,6 +72,9 @@
/** Default interval between two alerts (a,b) for considering them correlated */
#define DEFAULT_BAYESIAN_CORRELATION_INTERVAL 1200
/** Default interval in seconds between an invocation of the thread for parsing XML manual correlations and the next one */
#define DEFAULT_MANUAL_CORRELATIONS_PARSING_INTERVAL 120
/** Default interval of validity in seconds for an entry in the cache of correlated alerts */
#define DEFAULT_BAYESIAN_CORRELATION_CACHE_VALIDITY 600
@ -167,6 +170,9 @@ typedef struct
/** Default maximum interval, in seconds, between two alerts for being considered in the same cluster */
unsigned long clusterMaxAlertInterval;
/** Interval in seconds between an invocation of the thread for parsing XML manual correlations and the next one */
unsigned long manualCorrelationsParsingInterval;
/** Interval in seconds for which an entry in the cache of correlated alerts is valid */
unsigned long bayesianCorrelationCacheValidity;