Starting to support PostgreSQL for logging output

TODO

@@ -3,8 +3,6 @@ AVERAGE/HIGH PRIORITY:
 ======================
 
 - Full PostgreSQL support for output db
-- Redefine function names
-- Errno
 - Web interface
 - Code profiling
 - Comment all the code!!!
@@ -34,4 +32,5 @@ DONE:
 + Split bayesian correlation out of correlation.c
 + Clustering alerts with time constraints
 + Save clusters and correlations to db
++ Uniformed error messages format
 

alert_history.c

@@ -28,7 +28,6 @@
 
 PRIVATE AI_alert_event  *alerts_hash = NULL;
 
-
 /**
  * \brief  Free a hash table of alert events
  * \param  events  Hash table to be freed

alert_parser.c

@@ -19,17 +19,16 @@
 
 #include	"spp_ai.h"
 
+#include 	<pthread.h>
 #include	<stdio.h>
-#include	<unistd.h>
+#include	<sys/stat.h>
 #include	<time.h>
+#include	<unistd.h>
 
 #ifdef LINUX
 #include	<sys/inotify.h>
 #endif
 
-#include	<sys/stat.h>
-#include 	<pthread.h>
-
 /** \defgroup alert_parser Parse the alert log into binary structures
  * @{ */
 

bayesian.c

@@ -19,8 +19,8 @@
 
 #include	"spp_ai.h"
 
-#include	<time.h>
 #include	<math.h>
+#include	<time.h>
 
 /** \defgroup correlation Module for the correlation of hyperalerts
  * @{ */

build.sh

@@ -1,21 +0,0 @@
-#!/bin/sh
-
-OUTDIR=${HOME}/local/snort/lib/snort_dynamicpreprocessor
-LIBSF=libsf_ai_preproc
-
-make clean
-make
-chmod +x ./${LIBSF}.la
-cp ./${LIBSF}.la ${OUTDIR}
-cp .libs/${LIBSF}.a ${OUTDIR}
-cp .libs/${LIBSF}.so.0.0.0 ${OUTDIR}
-cd ${OUTDIR}
-
-if [ ! -f ${LIBSF}.so.0 ]; then
-	ln -sf ${LIBSF}.so.0.0.0 ${LIBSF}.so.0
-fi
-
-if [ ! -f ${LIBSF}.so ]; then
-	ln -sf ${LIBSF}.so.0.0.0 ${LIBSF}.so
-fi
-

cluster.c

@@ -19,12 +19,11 @@
 
 #include	"spp_ai.h"
 
+#include	<limits.h>
+#include	<math.h>
+#include 	<pthread.h>
 #include	<stdio.h>
 #include	<unistd.h>
-#include	<math.h>
-#include	<limits.h>
-#include	<errno.h>
-#include 	<pthread.h>
 
 /** \defgroup cluster Manage the clustering of alarms
  * @{ */

correlation.c

@@ -19,17 +19,16 @@
 
 #include	"spp_ai.h"
 
+#include	<alloca.h>
+#include	<libxml/xmlreader.h>
+#include	<math.h>
+#include 	<pthread.h>
 #include	<stdio.h>
 #include	<stdlib.h>
 #include	<string.h>
 #include	<unistd.h>
-#include	<time.h>
-#include	<math.h>
-#include	<errno.h>
-#include	<alloca.h>
 #include	<sys/stat.h>
-#include 	<pthread.h>
+#include	<time.h>
-#include	<libxml/xmlreader.h>
 
 #ifdef 	HAVE_LIBGVC
 	#include	<gvc.h>

db.c

@@ -22,9 +22,9 @@
 
 #include	"db.h"
 
-#include	<unistd.h>
-#include	<time.h>
 #include 	<pthread.h>
+#include	<time.h>
+#include	<unistd.h>
 
 /** \defgroup db Manage alerts on a database
  * @{ */

db.h

@@ -79,11 +79,11 @@
 #endif
 
 	void*          DB_init();
-	unsigned long  DB_escape_string();
+	unsigned long  DB_escape_string ( char **to, const char *from, unsigned long length );
 	void           DB_close();
 
 	void*          DB_out_init();
-	unsigned long  DB_out_escape_string();
+	unsigned long  DB_out_escape_string ( char **to, const char *from, unsigned long length );
 	void           DB_out_close();
 
 	#endif

outdb.c

@@ -28,6 +28,7 @@
 
 #include	"db.h"
 #include	"uthash.h"
+
 #include	<alloca.h>
 #include	<pthread.h>
 
@@ -212,8 +213,9 @@ AI_store_alert_to_db_thread ( void *arg )
 			if ( !( pkt_data = (unsigned char*) alloca ( 2 * (pkt->pkt->pcap_header->len + pkt->pkt->payload_size) + 1 )))
 				AI_fatal_err ( "Fatal dynamic memory allocation error", __FILE__, __LINE__ );
 
-			DB_out_escape_string ( &pkt_data,
+			DB_out_escape_string (
-					pkt->pkt->pkt_data,
+					(char**) &pkt_data,
+					(const char*) pkt->pkt->pkt_data,
 					pkt->pkt->pcap_header->len + pkt->pkt->payload_size );
 
 			memset ( query, 0, sizeof ( query ));

postgresql.c

@@ -20,9 +20,10 @@
 #include	"spp_ai.h"
 #ifdef 	HAVE_LIBPQ
 
+#include	"db.h"
+
 #include	<alloca.h>
 #include	<postgresql/libpq-fe.h>
-#include	"db.h"
 
 /** \defgroup postgresql Module for the interface with a PostgreSQL DBMS
  * @{ */
@@ -58,7 +59,7 @@ __postgresql_do_init ( PGconn **__DB, BOOL is_out )
 		  ((config->dbpass) ? strlen ( config->dbpass ) : 0) +
 		  ((config->dbname) ? strlen ( config->dbname ) : 0)) + 100;
 
-	if ( postgresql_is_init ( *__DB ))
+	if ( __postgresql_is_init ( *__DB ))
 		return (void*) *__DB;
 	
 	if ( !( conninfo = (char*) alloca ( conninfo_max_length )))
@@ -135,39 +136,6 @@ __postgresql_do_query ( PGconn *__DB, const char *query )
 	return res;
 }
 
-int
-postgresql_num_rows ( PSQL_result *res )
-{
-	return PQntuples ( res->res );
-}
-
-char**
-postgresql_fetch_row ( PSQL_result *res )
-{
-	if ( (res->index++) >= PQntuples ( res->res ))
-		return NULL;
-
-	return res->rows[ res->index - 1];
-}
-
-void
-postgresql_free_result ( PSQL_result *res )
-{
-	int i, j, ntuples;
-
-	if ( res )
-	{
-		ntuples = PQntuples ( res->res );
-
-		for ( i=0; i < ntuples; i++ )
-			free ( res->rows[i] );
-		free ( res->rows );
-
-		PQclear ( res->res );
-		free ( res );
-	}
-}
-
 PRIVATE void
 __postgresql_do_close ( PGconn **__DB )
 {
@@ -201,6 +169,14 @@ postgresql_do_query ( const char *query )
 	return __postgresql_do_query ( db, query );
 }
 
+unsigned long
+postgresql_do_escape_string ( char **to, const char *from, unsigned long length )
+{
+	size_t out_len = 0;
+	*to = (char*) PQescapeByteaConn ( db, (const unsigned char* ) from, (size_t) length, &out_len );
+	return (unsigned long) out_len;
+}
+
 void
 postgresql_do_close ()
 {
@@ -227,12 +203,55 @@ postgresql_do_out_query ( const char *query )
 	return __postgresql_do_query ( outdb, query );
 }
 
+unsigned long
+postgresql_do_out_escape_string ( char **to, const char *from, unsigned long length )
+{
+	size_t out_len = 0;
+	*to = (char*) PQescapeByteaConn ( outdb, (const unsigned char* ) from, (size_t) length, &out_len );
+	return (unsigned long) out_len;
+}
+
 void
 postgresql_do_out_close ()
 {
 	__postgresql_do_close ( &outdb );
 }
 
+/* Functions working on result sets */
+
+int
+postgresql_num_rows ( PSQL_result *res )
+{
+	return PQntuples ( res->res );
+}
+
+char**
+postgresql_fetch_row ( PSQL_result *res )
+{
+	if ( (res->index++) >= PQntuples ( res->res ))
+		return NULL;
+
+	return res->rows[ res->index - 1];
+}
+
+void
+postgresql_free_result ( PSQL_result *res )
+{
+	int i, ntuples;
+
+	if ( res )
+	{
+		ntuples = PQntuples ( res->res );
+
+		for ( i=0; i < ntuples; i++ )
+			free ( res->rows[i] );
+		free ( res->rows );
+
+		PQclear ( res->res );
+		free ( res );
+	}
+}
+
 /* End of public functions */
 /***************************/
 

regex.c

@@ -17,12 +17,13 @@
  * =====================================================================================
  */
 
+#include	"uthash.h"
+
+#include	<alloca.h>
+#include	<regex.h>
 #include	<stdio.h>
 #include	<stdlib.h>
 #include	<string.h>
-#include	<alloca.h>
-#include	<regex.h>
-#include	"uthash.h"
 
 /** Compiled and cached regular expression entry */
 struct regex_cache_entry {

schemas/postgresql.sql

@@ -0,0 +1,66 @@
+DROP TABLE IF EXISTS ca_ipv4_headers CASCADE;
+CREATE TABLE ca_ipv4_headers (
+	ip_hdr_id     serial   primary key,
+	ip_tos        integer,
+	ip_len        integer,
+	ip_id         integer,
+	ip_ttl        integer,
+	ip_proto      integer,
+	ip_src_addr   varchar(32),
+	ip_dst_addr   varchar(32)
+);
+
+DROP TABLE IF EXISTS ca_tcp_headers CASCADE;
+CREATE TABLE ca_tcp_headers (
+	tcp_hdr_id     serial   primary key,
+	tcp_src_port   integer,
+	tcp_dst_port   integer,
+	tcp_seq        integer,
+	tcp_ack        integer,
+	tcp_flags      integer,
+	tcp_window     integer,
+	tcp_len        integer
+);
+
+DROP TABLE IF EXISTS ca_clustered_alerts CASCADE;
+CREATE TABLE ca_clustered_alerts (
+	cluster_id        serial   primary key,
+	clustered_srcip   varchar(255) default null,
+	clustered_dstip   varchar(255) default null,
+	clustered_srcport varchar(255) default null,
+	clustered_dstport varchar(255) default null
+);
+
+DROP TABLE IF EXISTS ca_alerts CASCADE;
+CREATE TABLE ca_alerts (
+	alert_id       serial    primary key,
+	gid            integer,
+	sid            integer,
+	rev            integer,
+	priority       integer,
+	description    varchar(255),
+	classification varchar(255),
+	timestamp      timestamp,
+	ip_hdr         integer    references ca_ipv4_headers(ip_hdr_id),
+	tcp_hdr        integer    references ca_tcp_headers(tcp_hdr_id),
+	cluster_id     integer default 0 references ca_clustered_alerts(cluster_id)
+);
+
+DROP TABLE IF EXISTS ca_packet_streams CASCADE;
+CREATE TABLE ca_packet_streams (
+	pkt_id         serial   primary key,
+	alert_id       integer  references ca_alerts(alert_id),
+	pkt_len        integer,
+	timestamp      timestamp,
+	content        oid
+);
+
+DROP TABLE IF EXISTS ca_correlated_alerts CASCADE;
+CREATE TABLE ca_correlated_alerts (
+	alert1            integer   references ca_alerts(alert_id),
+	alert2            integer   references ca_alerts(alert_id),
+	correlation_coeff real,
+
+	primary key(alert1, alert2)
+);
+

spp_ai.c

@@ -21,10 +21,10 @@
 #include "sfPolicyUserData.h"
 #include "sf_preproc_info.h"
 
+#include <errno.h>
+#include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
-#include <pthread.h>
-#include <errno.h>
 
 /** \defgroup spp_ai Main file for spp_ai module
  * @{ */

stream.c

@@ -19,12 +19,11 @@
 
 #include 	"spp_ai.h"
 
+#include 	<pthread.h>
 #include	<stdio.h>
 #include	<stdlib.h>
-#include 	<time.h>
 #include	<unistd.h>
-#include 	<pthread.h>
+#include 	<time.h>
-
 
 PRIVATE struct pkt_info *hash = NULL;
 PRIVATE time_t start_time = 0;