From 544daa31cc01d4f860997bdb81f38e79fb5b8052 Mon Sep 17 00:00:00 2001 From: BlackLight Date: Thu, 14 Oct 2010 02:45:31 +0200 Subject: [PATCH] Supporting manual (un)correlations from web interface --- Makefile.am | 4 + Makefile.in | 4 + TODO | 3 +- alert_parser.c | 20 +- cluster.c | 7 +- corr_rules/1-1394-12.xml | 2 +- corr_rules/1-1924-8.xml | 2 +- corr_rules/1-469-4.xml | 2 +- corr_rules/1-579-10.xml | 2 +- corr_rules/122-1-0.xml | 2 +- corr_rules/hyperalert.dtd | 4 +- correlation.c | 425 +++++++++++++++++++++++++++++-- htdocs/correlate.cgi | 178 +++++++++++++ htdocs/index.html | 225 ++++++++++++++-- htdocs/manual_correlations.dtd | 19 ++ htdocs/manual_correlations.xml | 6 + htdocs/manual_uncorrelations.xml | 6 + htdocs/pcap.cgi | 2 +- spp_ai.c | 58 +++-- spp_ai.h | 6 + 20 files changed, 904 insertions(+), 73 deletions(-) create mode 100755 htdocs/correlate.cgi create mode 100644 htdocs/manual_correlations.dtd create mode 100644 htdocs/manual_correlations.xml create mode 100644 htdocs/manual_uncorrelations.xml diff --git a/Makefile.am b/Makefile.am index d12de53..66f4205 100644 --- a/Makefile.am +++ b/Makefile.am @@ -56,7 +56,11 @@ fi mkdir -p "${SHARE_PREFIX}/htdocs/js" mkdir -p "${SHARE_PREFIX}/schemas" install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_uncorrelations.xml" "${SHARE_PREFIX}/htdocs" install -m 0755 "${PWD}/htdocs/pcap.cgi" "${SHARE_PREFIX}/htdocs" + install -m 0755 "${PWD}/htdocs/correlate.cgi" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/js/Curry-1.0.1.js" "${SHARE_PREFIX}/htdocs/js" install -m 0644 "${PWD}/htdocs/js/dracula_algorithms.js" "${SHARE_PREFIX}/htdocs/js" install -m 0644 "${PWD}/htdocs/js/dracula_graffle.js" "${SHARE_PREFIX}/htdocs/js" diff --git a/Makefile.in b/Makefile.in index 03d1867..148d0c0 100644 --- a/Makefile.in +++ b/Makefile.in @@ -832,7 +832,11 @@ fi mkdir -p "${SHARE_PREFIX}/htdocs/js" mkdir -p "${SHARE_PREFIX}/schemas" install -m 0644 "${PWD}/htdocs/index.html" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_correlations.dtd" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_correlations.xml" "${SHARE_PREFIX}/htdocs" + install -m 0644 "${PWD}/htdocs/manual_uncorrelations.xml" "${SHARE_PREFIX}/htdocs" install -m 0755 "${PWD}/htdocs/pcap.cgi" "${SHARE_PREFIX}/htdocs" + install -m 0755 "${PWD}/htdocs/correlate.cgi" "${SHARE_PREFIX}/htdocs" install -m 0644 "${PWD}/htdocs/js/Curry-1.0.1.js" "${SHARE_PREFIX}/htdocs/js" install -m 0644 "${PWD}/htdocs/js/dracula_algorithms.js" "${SHARE_PREFIX}/htdocs/js" install -m 0644 "${PWD}/htdocs/js/dracula_graffle.js" "${SHARE_PREFIX}/htdocs/js" diff --git a/TODO b/TODO index 6260f11..61bd719 100644 --- a/TODO +++ b/TODO @@ -2,13 +2,14 @@ AVERAGE/HIGH PRIORITY: ====================== +- XML::Simple dependancy +- Manual alert correlation from the web interface - Bayesian network - Modules for correlation coefficients - Code profiling - Comment all the code!!! - Neural network for computing k - Testing more scenarios, making more hyperalert models -- Manual alert correlation from the web interface ============= LOW PRIORITY: diff --git a/alert_parser.c b/alert_parser.c index 8021706..e9e8093 100644 --- a/alert_parser.c +++ b/alert_parser.c @@ -347,20 +347,20 @@ AI_file_alertparser_thread ( void* arg ) } else if ( preg_match ( "^([0-9]{2})/([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})\\.[0-9]+\\s+([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}):([0-9]{1,5})\\s*" "->\\s*([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}):([0-9]{1,5})", line, &matches, &nmatches ) > 0 ) { - stamp = time(NULL); + stamp = time (NULL); _tm = localtime ( &stamp ); ltime.year = (unsigned short) _tm->tm_year + 1900; - ltime.day = (unsigned short) strtoul ( matches[0], NULL, 10 ); - ltime.month = (unsigned short) strtoul ( matches[1], NULL, 10 ); + ltime.day = (unsigned short) strtoul ( matches[1], NULL, 10 ); + ltime.month = (unsigned short) strtoul ( matches[0], NULL, 10 ); ltime.hour = (unsigned short) strtoul ( matches[2], NULL, 10 ); ltime.min = (unsigned short) strtoul ( matches[3], NULL, 10 ); ltime.sec = (unsigned short) strtoul ( matches[4], NULL, 10 ); snprintf ( strtime, sizeof(strtime), "%02hu/%02hu/%04hu, %02hu:%02hu:%02hu", - ltime.day, ltime.month, ltime.year, ltime.hour, ltime.min, ltime.sec ); + ltime.month, ltime.day, ltime.year, ltime.hour, ltime.min, ltime.sec ); - strptime ( strtime, "%d/%m/%Y, %H:%M:%S", _tm ); + strptime ( strtime, "%m/%d/%Y, %H:%M:%S", _tm ); alert->timestamp = mktime ( _tm ); alert->ip_src_addr = inet_addr ( matches[5] ); @@ -376,20 +376,20 @@ AI_file_alertparser_thread ( void* arg ) } else if ( preg_match ( "^([0-9]{2})/([0-9]{2})-([0-9]{2}):([0-9]{2}):([0-9]{2})\\.[0-9]+\\s+([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})\\s*" "->\\s*([0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3}\\.[0-9]{1,3})", line, &matches, &nmatches ) > 0 ) { - stamp = time(NULL); + stamp = time (NULL); _tm = localtime ( &stamp ); ltime.year = (unsigned short) _tm->tm_year + 1900; - ltime.day = (unsigned short) strtoul ( matches[0], NULL, 10 ); - ltime.month = (unsigned short) strtoul ( matches[1], NULL, 10 ); + ltime.day = (unsigned short) strtoul ( matches[1], NULL, 10 ); + ltime.month = (unsigned short) strtoul ( matches[0], NULL, 10 ); ltime.hour = (unsigned short) strtoul ( matches[2], NULL, 10 ); ltime.min = (unsigned short) strtoul ( matches[3], NULL, 10 ); ltime.sec = (unsigned short) strtoul ( matches[4], NULL, 10 ); snprintf ( strtime, sizeof(strtime), "%02hu/%02hu/%04hu, %02hu:%02hu:%02hu", - ltime.day, ltime.month, ltime.year, ltime.hour, ltime.min, ltime.sec ); + ltime.month, ltime.day, ltime.year, ltime.hour, ltime.min, ltime.sec ); - strptime ( strtime, "%d/%m/%Y, %H:%M:%S", _tm ); + strptime ( strtime, "%m/%d/%Y, %H:%M:%S", _tm ); alert->timestamp = mktime ( _tm ); alert->ip_src_addr = inet_addr ( matches[5] ); diff --git a/cluster.c b/cluster.c index 8d85dbb..e307444 100644 --- a/cluster.c +++ b/cluster.c @@ -392,8 +392,7 @@ __AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp ) { AI_snort_alert *tmp; char ip[INET_ADDRSTRLEN]; - char timestamp[128]; - struct tm *_tm; + char *timestamp; for ( tmp = log; tmp; tmp = tmp->next ) { @@ -404,8 +403,8 @@ __AI_print_clustered_alerts ( AI_snort_alert *log, FILE *fp ) fprintf ( fp, "[Priority: %d]\n", tmp->priority ); - _tm = localtime ( &tmp->timestamp ); - strftime ( timestamp, sizeof ( timestamp ), "%a %b %d %Y, %H:%M:%S", _tm ); + timestamp = ctime ( &( tmp->timestamp )); + timestamp [ strlen ( timestamp ) - 1 ] = 0; fprintf ( fp, "[Grouped alerts: %d] [Starting from: %s]\n", tmp->grouped_alerts_count, timestamp ); if ( h_root[src_addr] && tmp->h_node[src_addr] ) diff --git a/corr_rules/1-1394-12.xml b/corr_rules/1-1394-12.xml index 736cdb6..111c84b 100644 --- a/corr_rules/1-1394-12.xml +++ b/corr_rules/1-1394-12.xml @@ -1,5 +1,5 @@ - + 1.1394.12 diff --git a/corr_rules/1-1924-8.xml b/corr_rules/1-1924-8.xml index 3963b99..18634d7 100644 --- a/corr_rules/1-1924-8.xml +++ b/corr_rules/1-1924-8.xml @@ -1,5 +1,5 @@ - + 1.1924.8 diff --git a/corr_rules/1-469-4.xml b/corr_rules/1-469-4.xml index 3d56a42..6f8f7d0 100644 --- a/corr_rules/1-469-4.xml +++ b/corr_rules/1-469-4.xml @@ -1,5 +1,5 @@ - + 1.469.4 diff --git a/corr_rules/1-579-10.xml b/corr_rules/1-579-10.xml index 186bd6a..a074af7 100644 --- a/corr_rules/1-579-10.xml +++ b/corr_rules/1-579-10.xml @@ -1,5 +1,5 @@ - + 1.579.10 diff --git a/corr_rules/122-1-0.xml b/corr_rules/122-1-0.xml index a7ba04e..e7e46ac 100644 --- a/corr_rules/122-1-0.xml +++ b/corr_rules/122-1-0.xml @@ -1,5 +1,5 @@ - + 122.1.0 diff --git a/corr_rules/hyperalert.dtd b/corr_rules/hyperalert.dtd index e691fc5..c9e031a 100644 --- a/corr_rules/hyperalert.dtd +++ b/corr_rules/hyperalert.dtd @@ -1,7 +1,9 @@ + + + ]> diff --git a/correlation.c b/correlation.c index a6a55b4..9a7f32e 100644 --- a/correlation.c +++ b/correlation.c @@ -41,12 +41,32 @@ #error "libxml2 reader not enabled\n" #endif -/** Enumeration for the types of XML tags */ -enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, TAG_NUM }; +/** Enumeration for the types of hyperalert XML tags */ +enum { inHyperAlert, inSnortIdTag, inPreTag, inPostTag, HYP_TAG_NUM }; -PRIVATE AI_hyperalert_info *hyperalerts = NULL; -PRIVATE AI_snort_alert *alerts = NULL; -PRIVATE AI_alert_correlation *correlation_table = NULL; +/** Enumeration for the types of manual correlations XML tags */ +enum { inCorrelation, inCorrelations, inFromTag, inToTag, MAN_TAG_NUM }; + +typedef struct { + int from_gid; + int from_sid; + int from_rev; + int to_gid; + int to_sid; + int to_rev; +} AI_alert_type_pair_key; + +typedef struct { + AI_alert_type_pair_key key; + enum { manuallyNone, manuallyCorrelated, manuallyNotCorrelated } corr_type; + UT_hash_handle hh; +} AI_alert_type_pair; + +PRIVATE AI_hyperalert_info *hyperalerts = NULL; +PRIVATE AI_snort_alert *alerts = NULL; +PRIVATE AI_alert_correlation *correlation_table = NULL; +PRIVATE AI_alert_type_pair *manual_correlations = NULL; +PRIVATE AI_alert_type_pair *manual_uncorrelations = NULL; PRIVATE pthread_mutex_t mutex; /** @@ -83,10 +103,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp ) dst_port1[10], src_port2[10], dst_port2[10], - timestamp1[40], - timestamp2[40]; - - struct tm *t1, *t2; + *timestamp1, + *timestamp2; if ( !corr ) return; @@ -97,8 +115,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp ) snprintf ( src_port1, sizeof ( src_port1 ), "%d", ntohs ( corr->key.a->tcp_src_port )); snprintf ( dst_port1, sizeof ( dst_port1 ), "%d", ntohs ( corr->key.a->tcp_dst_port )); - t1 = localtime ( &(corr->key.a->timestamp )); - strftime ( timestamp1, sizeof ( timestamp1 ), "%a %b %d %Y, %H:%M:%S", t1 ); + timestamp1 = ctime ( &(corr->key.a->timestamp ) ); + timestamp1 [ strlen ( timestamp1 ) - 1 ] = 0; inet_ntop ( AF_INET, &(corr->key.b->ip_src_addr), src_addr2, INET_ADDRSTRLEN ); inet_ntop ( AF_INET, &(corr->key.b->ip_dst_addr), dst_addr2, INET_ADDRSTRLEN ); @@ -106,8 +124,8 @@ __AI_correlated_alerts_to_dot ( AI_alert_correlation *corr, FILE *fp ) snprintf ( src_port2, sizeof ( src_port2 ), "%d", ntohs ( corr->key.b->tcp_src_port )); snprintf ( dst_port2, sizeof ( dst_port2 ), "%d", ntohs ( corr->key.b->tcp_dst_port )); - t2 = localtime ( &(corr->key.b->timestamp )); - strftime ( timestamp2, sizeof ( timestamp2 ), "%a %b %d %Y, %H:%M:%S", t2 ); + timestamp2 = ctime ( &(corr->key.b->timestamp ) ); + timestamp2 [ strlen ( timestamp2 ) - 1 ] = 0; fprintf ( fp, "\t\"[%d.%d.%d] %s\\n" @@ -178,12 +196,18 @@ __AI_correlated_alerts_to_json () fprintf ( fp, "{\n" "\t\"id\": %lu,\n" + "\t\"snortSID\": \"%u\",\n" + "\t\"snortGID\": \"%u\",\n" + "\t\"snortREV\": \"%u\",\n" "\t\"label\": \"%s\",\n" "\t\"date\": \"%s\",\n" "\t\"clusteredAlertsCount\": %u,\n" "\t\"from\": \"%s:%s\",\n" "\t\"to\": \"%s:%s\"", alert_iterator->alert_id, + alert_iterator->sid, + alert_iterator->gid, + alert_iterator->rev, alert_iterator->desc, strtime, alert_iterator->grouped_alerts_count, @@ -204,7 +228,7 @@ __AI_correlated_alerts_to_json () AI_fatal_err ( "Fatal dynamic memory allocation", __FILE__, __LINE__ ); } - memset ( encoded_pkt, 0, 4*pkt_len + 1 ); + memset ( encoded_pkt, 0, 4*pkt_len + 1 ); base64_encode ( (const char*) pkt_iterator->pkt->pkt_data, @@ -617,6 +641,348 @@ __AI_kb_correlation_coefficient ( AI_snort_alert *a, AI_snort_alert *b ) } /* ----- end of function __AI_kb_correlation_coefficient ----- */ +/** + * \brief Parse the manual specified correlations from XML file(s) and fills the hash table + */ + +PRIVATE void* +__AI_manual_correlations_parsing_thread ( void *arg ) +{ + unsigned int i = 0; + char manual_correlations_xml[1060] = { 0 }, + manual_uncorrelations_xml[1060] = { 0 }; + struct stat st; + xmlTextReaderPtr xml; + const xmlChar *tagname; + AI_alert_type_pair_key key; + AI_alert_type_pair *pair = NULL, + *found = NULL; + BOOL xml_flags[MAN_TAG_NUM] = { false }; + + while ( 1 ) + { + /* Cleanup tables */ + while ( manual_correlations ) + { + pair = manual_correlations; + HASH_DEL ( manual_correlations, pair ); + free ( pair ); + } + + while ( manual_uncorrelations ) + { + pair = manual_uncorrelations; + HASH_DEL ( manual_uncorrelations, pair ); + free ( pair ); + } + + pair = NULL; + memset ( &key, 0, sizeof ( key )); + + snprintf ( manual_correlations_xml, + sizeof ( manual_correlations_xml ), + "%s/manual_correlations.xml", config->webserv_dir ); + + snprintf ( manual_uncorrelations_xml, + sizeof ( manual_uncorrelations_xml ), + "%s/manual_uncorrelations.xml", config->webserv_dir ); + + if ( stat ( manual_correlations_xml, &st ) < 0 ) + { + pthread_exit ((void*) 0); + return (void*) 0; + } + + if ( stat ( manual_uncorrelations_xml, &st ) < 0 ) + { + pthread_exit ((void*) 0); + return (void*) 0; + } + + LIBXML_TEST_VERSION + + /* Check manual correlations */ + if ( !( xml = xmlReaderForFile ( manual_correlations_xml, NULL, 0 ))) + { + pthread_exit ((void*) 0); + return (void*) 0; + } + + while ( xmlTextReaderRead ( xml )) + { + if ( !( tagname = xmlTextReaderConstName ( xml ))) + continue; + + if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) + { + if ( !strcasecmp ((const char*) tagname, "correlations" )) + { + if ( xml_flags[inCorrelations] ) + { + AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelations] = true; + } + } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { + if ( xml_flags[inCorrelation] ) + { + AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelation] = true; + } + } else if ( !strcasecmp ((const char*) tagname, "from" )) { + xml_flags[inFromTag] = true; + + key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; + key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; + key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; + + /* If this is a new pair, allocate the memory */ + if ( pair == NULL ) + { + if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + pair->corr_type = manuallyCorrelated; + } else { + /* Otherwise, add the pair to the hash, if it's not already there */ + pair->key = key; + HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found ); + + if ( !found ) + { + HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair ); + } + + pair = NULL; + memset ( &key, 0, sizeof ( key )); + } + } else if ( !strcasecmp ((const char*) tagname, "to" )) { + xml_flags[inToTag] = true; + + key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; + key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; + key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; + + /* If this is a new pair, allocate the memory */ + if ( pair == NULL ) + { + if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + pair->corr_type = manuallyCorrelated; + } else { + /* Otherwise, add the pair to the hash, if it's not already there */ + pair->key = key; + HASH_FIND ( hh, manual_correlations, &key, sizeof ( key ), found ); + + if ( !found ) + { + HASH_ADD ( hh, manual_correlations, key, sizeof ( key ), pair ); + } + + pair = NULL; + memset ( &key, 0, sizeof ( key )); + } + } else { + AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); + } + } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { + if ( !strcasecmp ((const char*) tagname, "correlations" )) + { + if ( !xml_flags[inCorrelations] ) + { + AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelations] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { + if ( !xml_flags[inCorrelation] ) + { + AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelation] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "from" )) { + if ( !xml_flags[inFromTag] ) + { + AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inFromTag] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "to" )) { + if ( !xml_flags[inToTag] ) + { + AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inToTag] = false; + } + } else { + AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); + } + } + } + + xmlFreeTextReader ( xml ); + xmlCleanupParser(); + + for ( i=0; i < MAN_TAG_NUM; i++ ) + { + xml_flags[i] = false; + } + + /* Check manual un-correlations */ + if ( !( xml = xmlReaderForFile ( manual_uncorrelations_xml, NULL, 0 ))) + { + pthread_exit ((void*) 0); + return (void*) 0; + } + + while ( xmlTextReaderRead ( xml )) + { + if ( !( tagname = xmlTextReaderConstName ( xml ))) + continue; + + if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_ELEMENT ) + { + if ( !strcasecmp ((const char*) tagname, "correlations" )) + { + if ( xml_flags[inCorrelations] ) + { + AI_fatal_err ( "Tag 'correlations' opened twice in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelations] = true; + } + } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { + if ( xml_flags[inCorrelation] ) + { + AI_fatal_err ( "Tag 'correlation' opened twice in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelation] = true; + } + } else if ( !strcasecmp ((const char*) tagname, "from" )) { + xml_flags[inFromTag] = true; + + key.from_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; + key.from_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; + key.from_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; + + /* If this is a new pair, allocate the memory */ + if ( pair == NULL ) + { + if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + pair->corr_type = manuallyNotCorrelated; + } else { + /* Otherwise, add the pair to the hash, if it's not already there */ + pair->key = key; + HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found ); + + if ( !found ) + { + HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair ); + } + + pair = NULL; + memset ( &key, 0, sizeof ( key )); + } + } else if ( !strcasecmp ((const char*) tagname, "to" )) { + xml_flags[inToTag] = true; + + key.to_gid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "gid" ), NULL, 10 ) : 0; + key.to_sid = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "sid" ), NULL, 10 ) : 0; + key.to_rev = (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" )) ? + strtol (( const char* ) xmlTextReaderGetAttribute ( xml, (const xmlChar*) "rev" ), NULL, 10 ) : 0; + + /* If this is a new pair, allocate the memory */ + if ( pair == NULL ) + { + if ( !( pair = ( AI_alert_type_pair* ) malloc ( sizeof ( AI_alert_type_pair )))) + { + AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); + } + + pair->corr_type = manuallyNotCorrelated; + } else { + /* Otherwise, add the pair to the hash, if it's not already there */ + pair->key = key; + HASH_FIND ( hh, manual_uncorrelations, &key, sizeof ( key ), found ); + + if ( !found ) + { + HASH_ADD ( hh, manual_uncorrelations, key, sizeof ( key ), pair ); + } + + pair = NULL; + memset ( &key, 0, sizeof ( key )); + } + } else { + AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); + } + } else if ( xmlTextReaderNodeType ( xml ) == XML_READER_TYPE_END_ELEMENT ) { + if ( !strcasecmp ((const char*) tagname, "correlations" )) + { + if ( !xml_flags[inCorrelations] ) + { + AI_fatal_err ( "Tag 'correlations' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelations] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "correlation" )) { + if ( !xml_flags[inCorrelation] ) + { + AI_fatal_err ( "Tag 'correlation' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inCorrelation] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "from" )) { + if ( !xml_flags[inFromTag] ) + { + AI_fatal_err ( "Tag 'from' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inFromTag] = false; + } + } else if ( !strcasecmp ((const char*) tagname, "to" )) { + if ( !xml_flags[inToTag] ) + { + AI_fatal_err ( "Tag 'to' closed but never opened in manual correlations XML file", __FILE__, __LINE__ ); + } else { + xml_flags[inToTag] = false; + } + } else { + AI_fatal_err ( "Unrecognized tag in manual correlations XML file", __FILE__, __LINE__ ); + } + } + } + + xmlFreeTextReader ( xml ); + xmlCleanupParser(); + sleep ( config->manualCorrelationsParsingInterval ); + } + + pthread_exit ((void*) 0); + return (void*) 0; +} /* ----- end of function __AI_manual_correlations_parsing_thread ----- */ + + /** * \brief Substitute the macros in hyperalert pre-conditions and post-conditions with their associated values * \param alert Reference to the hyperalert to work on @@ -719,7 +1085,7 @@ __AI_hyperalert_from_XML ( AI_hyperalert_key key ) { char hyperalert_file[1024] = {0}; char snort_id[1024] = {0}; - BOOL xmlFlags[TAG_NUM] = { false }; + BOOL xmlFlags[HYP_TAG_NUM] = { false }; struct stat st; xmlTextReaderPtr xml; const xmlChar *tagname, *tagvalue; @@ -867,13 +1233,18 @@ AI_alert_correlation_thread ( void *arg ) AI_alert_correlation_key corr_key; AI_alert_correlation *corr = NULL; + AI_alert_type_pair_key pair_key; + AI_alert_type_pair *pair = NULL, + *unpair = NULL; + AI_hyperalert_key key; AI_hyperalert_info *hyp = NULL; AI_snort_alert *alert_iterator = NULL, *alert_iterator2 = NULL; - pthread_t db_thread; + pthread_t db_thread, + manual_corr_thread; #ifdef HAVE_LIBGVC char corr_png_file[4096] = { 0 }; @@ -883,6 +1254,12 @@ AI_alert_correlation_thread ( void *arg ) pthread_mutex_init ( &mutex, NULL ); + /* Start the thread for parsing manual correlations from XML */ + if ( pthread_create ( &manual_corr_thread, NULL, __AI_manual_correlations_parsing_thread, NULL ) != 0 ) + { + AI_fatal_err ( "Failed to create the manual correlations parsing thread", __FILE__, __LINE__ ); + } + while ( 1 ) { sleep ( config->correlationGraphInterval ); @@ -955,6 +1332,7 @@ AI_alert_correlation_thread ( void *arg ) __AI_correlation_table_cleanup(); correlation_table = NULL; + /* Fill the table of correlated alerts */ for ( alert_iterator = alerts; alert_iterator; alert_iterator = alert_iterator->next ) { for ( alert_iterator2 = alerts; alert_iterator2; alert_iterator2 = alert_iterator2->next ) @@ -1026,12 +1404,23 @@ AI_alert_correlation_thread ( void *arg ) /* Find correlated alerts */ for ( corr = correlation_table; corr; corr = ( AI_alert_correlation* ) corr->hh.next ) { - if ( corr->correlation >= corr_threshold && + pair_key.from_sid = corr->key.a->sid; + pair_key.from_gid = corr->key.a->gid; + pair_key.from_rev = corr->key.a->rev; + pair_key.to_sid = corr->key.b->sid; + pair_key.to_gid = corr->key.b->gid; + pair_key.to_rev = corr->key.b->rev; + + HASH_FIND ( hh, manual_correlations, &pair_key, sizeof ( pair_key ), pair ); + HASH_FIND ( hh, manual_uncorrelations, &pair_key, sizeof ( pair_key ), unpair ); + + if ( !unpair && ( pair || ( + corr->correlation >= corr_threshold && corr_threshold != 0.0 && corr->key.a->timestamp <= corr->key.b->timestamp && ! ( corr->key.a->gid == corr->key.b->gid && corr->key.a->sid == corr->key.b->sid && - corr->key.a->rev == corr->key.b->rev )) + corr->key.a->rev == corr->key.b->rev )))) { if ( !( corr->key.a->derived_alerts = ( AI_snort_alert** ) realloc ( corr->key.a->derived_alerts, (++corr->key.a->n_derived_alerts) * sizeof ( AI_snort_alert* )))) AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ ); diff --git a/htdocs/correlate.cgi b/htdocs/correlate.cgi new file mode 100755 index 0000000..38fc885 --- /dev/null +++ b/htdocs/correlate.cgi @@ -0,0 +1,178 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use XML::Simple; +use Data::Dumper; +use Env qw(QUERY_STRING DOCUMENT_ROOT); + +exit 1 if ( !$QUERY_STRING ); + +$QUERY_STRING =~ /action=([a-z]+)/; +my $action = $1; + +$QUERY_STRING =~ /from_sid=([0-9]+)/; +my $from_sid = $1; + +$QUERY_STRING =~ /from_gid=([0-9]+)/; +my $from_gid = $1; + +$QUERY_STRING =~ /from_rev=([0-9]+)/; +my $from_rev = $1; + +$QUERY_STRING =~ /to_sid=([0-9]+)/; +my $to_sid = $1; + +$QUERY_STRING =~ /to_gid=([0-9]+)/; +my $to_gid = $1; + +$QUERY_STRING =~ /to_rev=([0-9]+)/; +my $to_rev = $1; + +exit 1 unless ( + defined ( $action ) && + defined ( $from_sid ) && + defined ( $from_gid ) && + defined ( $from_rev ) && + defined ( $to_sid ) && + defined ( $to_gid ) && + defined ( $to_rev )); + +my $xml = new XML::Simple ( forcearray => 1 ); +my $corr_data = $xml->XMLin ( "$DOCUMENT_ROOT/manual_correlations.xml" ); +my $uncorr_data = $xml->XMLin ( "$DOCUMENT_ROOT/manual_uncorrelations.xml" ); + +if ( $action eq 'add' ) +{ + # Check if 'correlation' already contains this item + for my $node ( @{$corr_data->{'correlation'}} ) + { + my $from = @{$node->{'from'}}[0]; + my $to = @{$node->{'to'}}[0]; + + exit 1 if ( + $from->{'sid'} eq $from_sid && + $from->{'gid'} eq $from_gid && + $from->{'rev'} eq $from_rev && + $to->{'sid'} eq $to_sid && + $to->{'gid'} eq $to_gid && + $to->{'rev'} eq $to_rev + ); + } + + # If this node is in 'uncorrelated' alerts, remove it from there + if ( UNIVERSAL::isa ( $uncorr_data->{'correlation'}, "ARRAY" )) + { + for my $i ( 0..@{$uncorr_data->{'correlation'}}-1 ) + { + if ( defined ( @{$uncorr_data->{'correlation'}}[$i] )) + { + my $from = @{@{$uncorr_data->{'correlation'}}[$i]->{'from'}}[0]; + my $to = @{@{$uncorr_data->{'correlation'}}[$i]->{'to'}}[0]; + + splice ( @{$uncorr_data->{'correlation'}}, $i, 1 ) if ( + $from->{'sid'} eq $from_sid && + $from->{'gid'} eq $from_gid && + $from->{'rev'} eq $from_rev && + $to->{'sid'} eq $to_sid && + $to->{'gid'} eq $to_gid && + $to->{'rev'} eq $to_rev + ); + } + } + } + + my %hash = ( + 'to' => [ + { + 'sid' => $to_sid, + 'gid' => $to_gid, + 'rev' => $to_rev, + } + ], + 'from' => [ + { + 'sid' => $from_sid, + 'gid' => $from_gid, + 'rev' => $from_rev + } + ] + ); + + push @{$corr_data->{'correlation'}}, \%hash; +} elsif ( $action eq 'remove' ) { + # Check if 'un-correlation' already contains this item + for my $node ( @{$uncorr_data->{'correlation'}} ) + { + my $from = @{$node->{'from'}}[0]; + my $to = @{$node->{'to'}}[0]; + + exit 1 if ( + $from->{'sid'} eq $from_sid && + $from->{'gid'} eq $from_gid && + $from->{'rev'} eq $from_rev && + $to->{'sid'} eq $to_sid && + $to->{'gid'} eq $to_gid && + $to->{'rev'} eq $to_rev + ); + } + + # If this node is in 'correlated' alerts, remove it from there + if ( UNIVERSAL::isa ( $corr_data->{'correlation'}, "ARRAY" )) + { + for my $i ( 0..@{$corr_data->{'correlation'}}-1 ) + { + if ( defined ( @{$corr_data->{'correlation'}}[$i] )) + { + my $from = @{@{$corr_data->{'correlation'}}[$i]->{'from'}}[0]; + my $to = @{@{$corr_data->{'correlation'}}[$i]->{'to'}}[0]; + + splice ( @{$corr_data->{'correlation'}}, $i, 1 ) if ( + $from->{'sid'} eq $from_sid && + $from->{'gid'} eq $from_gid && + $from->{'rev'} eq $from_rev && + $to->{'sid'} eq $to_sid && + $to->{'gid'} eq $to_gid && + $to->{'rev'} eq $to_rev + ); + } + } + } + + my %hash = ( + 'to' => [ + { + 'sid' => $to_sid, + 'gid' => $to_gid, + 'rev' => $to_rev + } + ], + 'from' => [ + { + 'sid' => $from_sid, + 'gid' => $from_gid, + 'rev' => $from_rev + } + ] + ); + + push @{$uncorr_data->{'correlation'}}, \%hash; +} + +my $xml_corr_out = $xml->XMLout ( $corr_data, RootName => "correlations" ); +my $xml_uncorr_out = $xml->XMLout ( $uncorr_data, RootName => "correlations" ); + +open OUT, "> $DOCUMENT_ROOT/manual_correlations.xml" or exit 1; +print OUT "\n"; +print OUT "\n\n"; +print OUT $xml_corr_out."\n"; +close OUT; + +open OUT, "> $DOCUMENT_ROOT/manual_uncorrelations.xml" or exit 1; +print OUT "\n"; +print OUT "\n\n"; +print OUT $xml_uncorr_out."\n"; +close OUT; + +print "Content-Type: text/html\n\n"; + diff --git a/htdocs/index.html b/htdocs/index.html index 5984ca6..8d94949 100644 --- a/htdocs/index.html +++ b/htdocs/index.html @@ -1,19 +1,10 @@ @@ -37,12 +39,66 @@