diff --git a/AUTHORS b/AUTHORS index e69de29..80b5000 100644 --- a/AUTHORS +++ b/AUTHORS @@ -0,0 +1,4 @@ +Author : Fabio "BlackLight" Manganiello +Email : +Website: http://0x00.ath.cx + diff --git a/ChangeLog b/ChangeLog index 203247c..777d3db 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,6 @@ +2010-14-09 Fabio "BlackLight" Manganiello + * README: Documentation added + 2010-14-09 Fabio "BlackLight" Manganiello * correlation.c: Correlation completed, runtime substitution of macros +ANY_ADDR+ and +ANY_PORT+ for runtime matching, runtime expansion of IP diff --git a/README b/README index e69de29..df2997e 100644 --- a/README +++ b/README @@ -0,0 +1,301 @@ +============================================================================ + ,,_ ____ _ _ ___ + o" )~ / ___| _ __ ___ _ __| |_ / \ |_ _| + '''' \___ \| '_ \ / _ \| '__| __| / _ \ | | + ___) | | | | (_) | | | |_ / ___ \ | | + |____/|_| |_|\___/|_| \__| /_/ \_\___| + + + _ __ _ __ ___ _ __ _ __ ___ ___ ___ ___ ___ ___ _ __ + | '_ \| '__/ _ \ '_ \| '__/ _ \ / __/ _ \/ __/ __|/ _ \| '__| + | |_) | | | __/ |_) | | | (_) | (_| __/\__ \__ \ (_) | | + | .__/|_| \___| .__/|_| \___/ \___\___||___/___/\___/|_| + |_| |_| + + ~ A REALLY smart preprocessor module for Snort ~ + by BlackLight , http://0x00.ath.cx +============================================================================ + + +This document describes the AI preprocessor module for Snort. +It also describes how to get it, install it, configure it and use it correctly. + +Table of contents: + 1. What's Snort AI preprocessor + 2. How to get Snort AI preprocessor + 3. Installation + 3.1 Dependancies + 3.2 Configure options + 4. Basic configuration + 5. Correlation rules + 6. Additional documentation + + +=============================== +1. What's Snort AI preprocessor +=============================== + +Snort AI preprocessor is a preprocessor module for Snort whose purpose is making +the reading of Snort's alerts more comfortable, clustering false positive alarms +emphasizing their root cause in order to reduce log pollution, clustering +similar alerts in function of the type and hierarchies over IP addresses and +ports that can be decided by the user, depending on the kind of traffic and +topology of the network, and constructing the flows of a multi-step attack in +function of correlation rules between hyperalerts provided by the developer +itself, by third parts or created by the user itself, again, in function of the +scenario of the network. It will furthermore possible, in a close future, to +correlate the hyperalerts automatically, by self-learning on the base of the +acquired alerts. + + +=================================== +2. How to get Snort AI preprocessor +=================================== + +It it strongly suggested to get the latest and always-fresh release of Snort AI +preprocessor from GitHub -> http://github.com/BlackLight/Snort_AIPreproc + +git clone git://github.com/BlackLight/Snort_AIPreproc.git + +If git is not available on the machine or cannot be used, from the same page you +can also choose "download source" and download the source code in tar.gz format. + + +=============== +3. Installation +=============== + +The installation procedure is the usual one: + +$ ./configure +$ make +$ make install + +If you did not install Snort in /usr directory you may need to use the --prefix +option with configure for selecting the directory where you installed Snort (for + example ./configure --prefix=$HOME/local/snort). If the prefix was +specified correctly, and it actually points to the location where Snort was +installed, the module binaries should be placed in +$SNORT_DIR/lib/snort_dynamicpreprocessor after the installation, and +automatically loaded by Snort at the next start. Moreover, a new directory +named corr_rules will be created, in /etc/snort if the prefix was /usr or in +$SNORT_DIR/etc otherwise, containing XML files describing default correlation +rules provided by the developer. This set can be enriched in any moment with new +XML files, provided by third parts or created by the user itself, describing +more hyperalerts. + + +================ +3.1 Dependancies +================ + +Dependancies required for a correct compilation and configuration: + +- pthread (REQUIRED), used for running multiple threads inside of the module. On +a Debian-based system, install libpthread-dev if you don't already have it. + +- libxml2 (REQUIRED), used for parsing XML files from corr_rules directory. On a +Debian-based system, install libxml2-dev if you don't already have it. + +- libgraphviz (RECOMMANDED), used for generating PNG (and in future PS too) + files representing hyperalert correlation graphs from .dot files + generated from the software. You can remove this dependancy from the + compilation process by specifying --without-graphviz to ./configure, but in + this case you will have .dot files, not easily understandable by a human, + for representing correlation graphs, and you may need an external graph + rendering software for converting them in a more easily readable format. On + a Debian system, install libgraphviz-dev if you don't already have it. + +- libmysqlclient (OPTIONAL), used if you want to read alerts information saved +on MySQL DBMS, or enable MySQL support in the module. This option is disabled by +default (if not specified otherwise, the module will read the alerts from Snort + plain log files), and can be enabled by specifying the option +--with-mysql to ./configure. On a Debian-based system you may need to install +libmysqlclient-dev. + + +===================== +3.2 Configure options +===================== + +You can pass the following options to ./configure script before compiling: + +--with-mysql - Enables MySQL DBMS support into the module (it requires + libmysqlclient) + +--without-graphviz - Disables Graphviz support from the module, avoiding the +generation of PNG or PS files representing hyperalerts correlation as well + + +====================== +4. Basic configuration +====================== + +After installing the module in Snort installation directory a configuration for +this is required in snort.conf. A sample configuration may appear like the +following: + + +preprocessor ai: \ + hashtable_cleanup_interval 300 \ + tcp_stream_expire_interval 300 \ + alertfile "/home/youruser/local/snort/log/alert" \ + alert_clustering_interval 300 \ + correlation_graph_interval 300 \ + correlation_rules_dir "/your/snort/dir/etc/corr_rules" \ + correlated_alerts_dir "/your/snort/dir/log/correlated_alerts" \ + correlation_threshold_coefficient 0.5 \ + database ( type="mysql", name="snort", user="snortusr", password="snortpass", host="dbhost" ) \ + database_parsing_interval 30 \ + clusterfile "/your/snort/dir/log/clustered_alerts" \ + cluster ( class="dst_port", name="privileged_ports", range="1-1023" ) \ + cluster ( class="dst_port", name="unprivileged_ports", range="1024-65535" ) \ + cluster ( class="src_addr", name="local_net", range="192.168.1.0/24" ) \ + cluster ( class="src_addr", name="dmz_net", range="155.185.0.0/16" ) \ + cluster ( class="src_addr", name="vpn_net", range="10.8.0.0/24" ) \ + cluster ( class="dst_addr", name="local_net", range="192.168.1.0/24" ) \ + cluster ( class="dst_addr", name="dmz_net", range="155.185.0.0/16" ) \ + cluster ( class="dst_addr", name="vpn_net", range="10.8.0.0/24" ) + +The options are the following: + +- hashtable_cleanup_interval: The interval that should occur from the cleanup of +the hashtable of TCP streams and the next one (default if not specified: 300 + seconds) + +- tcp_stream_expire_interval: The interval that should occur for marking a TCP +stream as "expired", if no more packets are received inside of that and it's not +"marked" as suspicious (default if not specified: 300 seconds) + +- alertfile: The file where Snort saves its alerts, if they are saved to a file +and not to a database (default if not specified: /var/log/snort/alert) + +- alert_clustering_interval: The interval that should occur from the clustering +of the alerts in the log according to the provided clustering hierarchies and +the next one (default if not specified: 300 seconds) + +- correlation_graph_interval: The interval that should occur from the building +of the correlation graph between the clustered alerts and the next one (default + if not specified: 300 seconds) + +- correlation_rules_dir: Directory where the correlation rules are saved, as XML +files (default if not specified: /etc/snort/corr_rules) + +- correlated_alerts_dir: Directory where the information between correlated +alerts will be saved, as .dot files ready to be rendered as graphs and, if +libgraphviz support is enabled, as .png and .ps files as well (default if not + specified: /var/log/snort/clustered_alerts) + +- correlation_threshold_coefficient: The threshold the software uses for stating +two alerts are correlated is avg(correlation coefficient) + k * +std_deviation(correlation_coefficient). The value of k is specified through this +option, whose value is 0.5 by default, and is dependant on how "sensible" you +want the correlation algorithm. A value of k=0 means "consider all the couples +of alerts whose correlation coefficient is greater than the average one as +correlated" (negative values of k are allowed as well, but unless you know what +you're doing they're unrecommended, as some correlation constraints may appear +where no correlation exists). When the value of k raises also the threshold for +two alerts for being considered as correlated raises. A high value of k may just +lead to an empty correlation graph + +- database: If Snort saves its alerts to a database and the module was compiled + with database support (e.g. --with-mysql) this option specifies the + information for accessing that database. The fields in side are + -- type: DBMS to be used (so far only MySQL is supported) + -- name: Database name + -- user: Username for accessing the database + -- password: Password for accessing the database + -- host: Host holding the database + +- database_parsing_interval: The interval that should occur between a read of +the alerts from database and the next one (default if not specified: 30 seconds) + +- clusterfile: File where the clustered alerts will be saved by the module +(default if not specified: /var/log/snort/clustered_alerts) + +- cluster: Clustering hierarchy or list of hierarchies to be applied for +grouping similar alerts. This option needs to specify: + -- class: Class of the cluster node. It may be src_addr, dst_addr, src_port + or dst_port + -- name: Name for the clustering node + -- range: Range of the clustering node. It can include a single port or IP + address, an IP range (specified as subnet x.x.x.x/x), or a port + range (specified as xxx-xxx) + + +==================== +5. Correlation rules +==================== + +The hyperalert correlation rules are specified in $SNORT_DIR/etc/corr_rules +directory through a very simple XML syntax, and any user can add some new ones. +The files in there must be named like the Snort alert ID they want to model. For +example, if we want to model a TCP portscan alert (Snort ID: 122.1.0) as a +hyperalert, i.e. as an alert with pre-conditions and post-conditions to be +correlated to other alerts, then we need to create a file named 122-1-0.xml +inside $SNORT_DIR/etc/corr_rules with a content like the following: + + + + + + + 122.1.0 + (portscan) TCP Portscan + +
HostExists(+DST_ADDR+)
+ HasService(+DST_ADDR+, +ANY_PORT+) +
+ + +The tag is optional, same for
  and  if an alert has no
+pre-conditions and/or post-conditions, while the  tag is mandatory for
+identifying the hyperalert. In this case we say that the pre-condition for a TCP
+portscan  for  being  successful  is  that the host +DST_ADDR+ exists (the macro
+ +DST_ADDR+   will   automatically   be  expanded  at  runtime  and  substituted
+ with   the   target   address   of  the  portscan).  The  post-condition  of  a
+portscan  consists  in  the  attacker  knowing that +DST_ADDR+ runs a service on
++ANY_PORT+  (+ANY_PORT+  is another macro that will be expanded on runtime). The
+hyperalerts  model  in  corr_rules  are  the knowledge base used for correlating
+alerts triggered by Snort, the more information it has inside, the more accurate
+and  complete  the  correlation will be. The macros recognized and automatically
+expanded           from           these          XML          files          are
+
+
+- +SRC_ADDR+: The IP address triggering the alert
+- +DST_ADDR+: The target IP address in the alert
+- +SRC_PORT+: The port from which the alert was triggered
+- +DST_PORT+: The port on which the alert was triggered
+- +ANY_ADDR+: Identifies any IP address
+- +ANY_PORT+: Identifies any port
+
+
+The correlation between two alerts A and B is made comparing the post-conditions
+of  A  with  the  pre-conditions of B. If the correlaton coefficient computed in
+this  way  is  greater  than  a  certain  threshold (see "Basic configuration ->
+ correlation_threshold_coefficient")    then    the   alerts   are   marked   as
+correlated, i.e. the alert A determines the alert B. This supports an elementary
+reasoning  algorithm  for  doing inferences on the conditions. For example, if A
+has  the  post-condition  "HasService(+DST_ADDR+,  +ANY_PORT+)"  and  B  has the
+pre-condition "HasService(+DST_ADDR, 22)", a match between A and B is triggered.
+Same   if   A   has   "HostExists(10.8.0.0/24)"  as  post-condition  and  B  has
+"HostExists(10.8.0.1)" as pre-condition.
+
+There  is  no  fixed  semantics  for  the  the  predicates in pre-conditions and
+post-conditions,  any  predicates  can  be used. The only constraint is that the
+same  predicates have the same semantic and prototype in all of the hyperalerts.
+For  example,  if  HasService  has a prototype like "HasService(ip_addr, port)",
+then   all    the  hyperalerts  should  follow  this  prototype,  otherwise  the
+matching    would   fail.   Any  new  predicates  can  be  defined  as  well  in
+hyperalerts,      provided     that     it     respects     this     constraint.
+
+
+===========================
+6. Additional documentation
+===========================
+
+
+The  additional  doxygen-generated  documentation  over  the code, functions and
+structures  can  be  found  in doc/ directory of source code or in SNORT_DIR/doc
+after installation.
+
diff --git a/corr_rules/1-1394-12.xml b/corr_rules/1-1394-12.xml
index 77cca0f..736cdb6 100644
--- a/corr_rules/1-1394-12.xml
+++ b/corr_rules/1-1394-12.xml
@@ -3,11 +3,11 @@
 
 
 	1.1394.12
-	Shellcode x86 inc ecx noop
+	Shellcode x86 inc ecx NOOP
 
 	
HostExists(+DST_ADDR+)
HasService(+DST_ADDR+, +DST_PORT+)
- HasLocalAccess(+SRC_ADDR+, +DST_ADDR+) + HasRemoteAccess(+SRC_ADDR+, +DST_ADDR+)
diff --git a/correlation.c b/correlation.c index 3938fe2..7cca168 100644 --- a/correlation.c +++ b/correlation.c @@ -93,10 +93,11 @@ _AI_correlation_table_cleanup () * \brief Recursively write a flow of correlated alerts to a .dot file, ready for being rendered as graph * \param corr Correlated alerts * \param fp File pointer + * \param strong Boolean value set if the correlation between the alerts is 'strong' (greater than avg + 2*k*deviation) */ PRIVATE void -_AI_print_correlated_alerts ( AI_alert_correlation *corr, FILE *fp ) +_AI_print_correlated_alerts ( AI_alert_correlation *corr, FILE *fp, BOOL strong ) { char src_addr1[INET_ADDRSTRLEN], dst_addr1[INET_ADDRSTRLEN], @@ -134,57 +135,25 @@ _AI_print_correlated_alerts ( AI_alert_correlation *corr, FILE *fp ) fprintf ( fp, "\t\"[%d.%d.%d] %s\\n" - "%s%s%s:%s%s%s -> %s%s%s:%s%s%s\\n" + "%s:%s -> %s:%s\\n" "%s\\n" "(%d alerts grouped)\" -> " "\"[%d.%d.%d] %s\\n" - "%s%s%s:%s%s%s -> %s%s%s:%s%s%s\\n" + "%s:%s -> %s:%s\\n" "%s\\n" - "(%d alerts grouped)\";\n", + "(%d alerts grouped)\"%s;\n", corr->key.a->gid, corr->key.a->sid, corr->key.a->rev, corr->key.a->desc, - - ( corr->key.a->h_node[src_addr] ) ? "[" : "", - ( corr->key.a->h_node[src_addr] ) ? corr->key.a->h_node[src_addr]->label : src_addr1, - ( corr->key.a->h_node[src_addr] ) ? "]" : "", - - ( corr->key.a->h_node[src_port] ) ? "[" : "", - ( corr->key.a->h_node[src_port] ) ? corr->key.a->h_node[src_port]->label : src_port1, - ( corr->key.a->h_node[src_port] ) ? "]" : "", - - ( corr->key.a->h_node[dst_addr] ) ? "[" : "", - ( corr->key.a->h_node[dst_addr] ) ? corr->key.a->h_node[dst_addr]->label : dst_addr1, - ( corr->key.a->h_node[dst_addr] ) ? "]" : "", - - ( corr->key.a->h_node[dst_port] ) ? "[" : "", - ( corr->key.a->h_node[dst_port] ) ? corr->key.a->h_node[dst_port]->label : dst_port1, - ( corr->key.a->h_node[dst_port] ) ? "]" : "", - + src_addr1, src_port1, dst_addr1, dst_port1, timestamp1, corr->key.a->grouped_alarms_count, - corr->key.b->gid, corr->key.b->sid, corr->key.b->rev, corr->key.b->desc, - - ( corr->key.b->h_node[src_addr] ) ? "[" : "", - ( corr->key.b->h_node[src_addr] ) ? corr->key.b->h_node[src_addr]->label : src_addr2, - ( corr->key.b->h_node[src_addr] ) ? "]" : "", - - ( corr->key.b->h_node[src_port] ) ? "[" : "", - ( corr->key.b->h_node[src_port] ) ? corr->key.b->h_node[src_port]->label : src_port2, - ( corr->key.b->h_node[src_port] ) ? "]" : "", - - ( corr->key.b->h_node[dst_addr] ) ? "[" : "", - ( corr->key.b->h_node[dst_addr] ) ? corr->key.b->h_node[dst_addr]->label : dst_addr2, - ( corr->key.b->h_node[dst_addr] ) ? "]" : "", - - ( corr->key.b->h_node[dst_port] ) ? "[" : "", - ( corr->key.b->h_node[dst_port] ) ? corr->key.b->h_node[dst_port]->label : dst_port2, - ( corr->key.b->h_node[dst_port] ) ? "]" : "", - + src_addr2, src_port2, dst_addr2, dst_port2, timestamp2, - corr->key.b->grouped_alarms_count + corr->key.b->grouped_alarms_count, + strong ? "" : "[style=dotted]" ); } /* ----- end of function _AI_correlation_flow_to_file ----- */ @@ -718,27 +687,28 @@ AI_alert_correlation_thread ( void *arg ) { int i; struct stat st; - char corr_dot_file[4096] = { 0 }; + char corr_dot_file[4096] = { 0 }; - double avg_correlation = 0.0, - std_deviation = 0.0, - corr_threshold = 0.0; + double avg_correlation = 0.0, + std_deviation = 0.0, + corr_threshold = 0.0, + corr_strong_threshold = 0.0; - FILE *fp = NULL; + FILE *fp = NULL; AI_alert_correlation_key corr_key; - AI_alert_correlation *corr = NULL; + AI_alert_correlation *corr = NULL; AI_hyperalert_key key; - AI_hyperalert_info *hyp = NULL; + AI_hyperalert_info *hyp = NULL; - AI_snort_alert *alert_iterator = NULL, - *alert_iterator2 = NULL; + AI_snort_alert *alert_iterator = NULL, + *alert_iterator2 = NULL; #ifdef HAVE_LIBGVC - char corr_png_file[4096] = { 0 }; - GVC_t *gvc = NULL; - graph_t *g = NULL; + char corr_png_file[4096] = { 0 }; + GVC_t *gvc = NULL; + graph_t *g = NULL; #endif conf = (AI_config*) arg; @@ -858,6 +828,7 @@ AI_alert_correlation_thread ( void *arg ) std_deviation = sqrt ( std_deviation / (double) HASH_COUNT ( correlation_table )); corr_threshold = avg_correlation + ( conf->correlationThresholdCoefficient * std_deviation ); + corr_strong_threshold = avg_correlation + ( 2.0 * conf->correlationThresholdCoefficient * std_deviation ); snprintf ( corr_dot_file, sizeof ( corr_dot_file ), "%s/correlated_alerts.dot", conf->corr_alerts_dir ); if ( stat ( conf->corr_alerts_dir, &st ) < 0 ) @@ -877,8 +848,8 @@ AI_alert_correlation_thread ( void *arg ) /* Find correlated alerts */ for ( corr = correlation_table; corr; corr = ( AI_alert_correlation* ) corr->hh.next ) { - if ( corr->correlation >= avg_correlation + std_deviation && - avg_correlation + std_deviation != 0.0 && + if ( corr->correlation >= corr_threshold && + corr_threshold != 0.0 && corr->key.a->timestamp <= corr->key.b->timestamp && ! ( corr->key.a->gid == corr->key.b->gid && corr->key.a->sid == corr->key.b->sid && @@ -889,7 +860,7 @@ AI_alert_correlation_thread ( void *arg ) corr->key.a->derived_alerts[ corr->key.a->n_derived_alerts - 1 ] = corr->key.b; corr->key.b->previous_correlated = corr->key.a; - _AI_print_correlated_alerts ( corr, fp ); + _AI_print_correlated_alerts ( corr, fp, ( corr->correlation >= corr_strong_threshold )); } } diff --git a/spp_ai.h b/spp_ai.h index 7b81467..ae94115 100644 --- a/spp_ai.h +++ b/spp_ai.h @@ -40,7 +40,7 @@ #define DEFAULT_DATABASE_INTERVAL 30 /** Default interval in seconds for the thread clustering alerts */ -#define DEFAULT_ALERT_CLUSTERING_INTERVAL 3600 +#define DEFAULT_ALERT_CLUSTERING_INTERVAL 300 /** Default interval in seconds for running the graph correlation thread */ #define DEFAULT_ALERT_CORRELATION_INTERVAL 300 @@ -49,7 +49,7 @@ #define DEFAULT_ALERT_LOG_FILE "/var/log/snort/alert" /** Default path to Snort's clustered alerts file */ -#define DEFAULT_CLUSTER_LOG_FILE "/var/log/snort/cluster_alert" +#define DEFAULT_CLUSTER_LOG_FILE "/var/log/snort/clustered_alerts" /** Default path to alert correlation rules directory */ #define DEFAULT_CORR_RULES_DIR "/etc/snort/corr_rules"