From 20cc395510b14f1e61462142135beeaeedf6ce71 Mon Sep 17 00:00:00 2001 From: brectanus Date: Tue, 2 Sep 2008 23:10:36 +0000 Subject: [PATCH] Added mlogc source. --- CHANGES | 4 +- apache2/Makefile.in | 17 +- apache2/Makefile.win | 3 +- apache2/configure | 4 + apache2/configure.in | 1 + apache2/mlogc-src/INSTALL | 76 + apache2/mlogc-src/Makefile.in | 70 + apache2/mlogc-src/mlogc-batch-load.pl.in | 151 ++ apache2/mlogc-src/mlogc-default.conf | 91 ++ apache2/mlogc-src/mlogc.c | 1865 ++++++++++++++++++++++ apache2/mod_security2.c | 3 +- apache2/modsecurity.c | 9 - apache2/modsecurity.h | 39 +- apache2/msc_logging.c | 4 +- apache2/msc_release.c | 42 + apache2/msc_release.h | 67 + apache2/msc_test.c | 2 +- apache2/msc_util.c | 13 +- 18 files changed, 2385 insertions(+), 76 deletions(-) create mode 100644 apache2/mlogc-src/INSTALL create mode 100755 apache2/mlogc-src/Makefile.in create mode 100755 apache2/mlogc-src/mlogc-batch-load.pl.in create mode 100644 apache2/mlogc-src/mlogc-default.conf create mode 100644 apache2/mlogc-src/mlogc.c create mode 100644 apache2/msc_release.c create mode 100644 apache2/msc_release.h diff --git a/CHANGES b/CHANGES index e83c35f3..6cfe4f01 100644 --- a/CHANGES +++ b/CHANGES @@ -1,6 +1,8 @@ -31 Jul 2008 - trunk +02 Sep 2008 - trunk ------------------- + * Integrate mlogc source. + * Allow for disabling request body limit checks in phase:1. * Added transformations for processing parity for legacy protocols ported diff --git a/apache2/Makefile.in b/apache2/Makefile.in index 320b4152..29314190 100644 --- a/apache2/Makefile.in +++ b/apache2/Makefile.in @@ -3,16 +3,16 @@ MOD_SECURITY2 = mod_security2 apache2_config apache2_io apache2_util \ re re_operators re_actions re_tfns re_variables \ msc_logging msc_xml msc_multipart modsecurity msc_parsers msc_util msc_pcre \ - persist_dbm msc_reqbody pdf_protect msc_geo acmp msc_lua + persist_dbm msc_reqbody pdf_protect msc_geo acmp msc_lua msc_release MSC_TEST = re re_operators re_actions re_tfns re_variables \ msc_logging msc_xml msc_multipart modsecurity \ msc_parsers msc_util msc_pcre persist_dbm \ - msc_reqbody msc_geo acmp msc_lua + msc_reqbody msc_geo acmp msc_lua msc_release MOD_SECURITY2_H = re.h modsecurity.h msc_logging.h msc_multipart.h msc_parsers.h \ msc_pcre.h msc_util.h msc_xml.h persist_dbm.h apache2.h pdf_protect.h \ - msc_geo.h acmp.h utf8tables.h msc_lua.h + msc_geo.h acmp.h utf8tables.h msc_lua.h msc_release.h CC = @APXS_CC@ LIBTOOL = @APXS_LIBTOOL@ @@ -70,7 +70,7 @@ clean-extras: $(MAKE) -C $$dir clean; \ fi; \ done - @rm -rf ../tools/mlogc ../tools/mlogc-static + @rm -rf ../tools/mlogc ../tools/mlogc-batch-load.pl clean: clean-extras @rm -rf *.la *.lo *.o *.slo .libs msc_test msc-test-debug.log @@ -99,19 +99,12 @@ mod_security2.la: $(MOD_SECURITY2_H) *.c mlogc: @$(MAKE) -C mlogc-src mlogc \ && cp -p mlogc-src/mlogc ../tools \ + && cp -p mlogc-src/mlogc-batch-load.pl ../tools \ && echo \ && echo "Successfully built \"mlogc\" in ../tools." \ && echo "See: mlogc-src/INSTALL" \ && echo -mlogc-static: - @$(MAKE) -C mlogc-src static \ - && cp -p mlogc-src/mlogc ../tools/mlogc-static \ - && echo \ - && echo "Successfully built \"mlogc-static\" in ../tools." \ - && echo "See: mlogc-src/INSTALL" \ - && echo - ### Experimental Test Framework (*NIX only right now) msc_test.lo: msc_test.c $(LIBTOOL) --mode=compile $(CC) $(APXS_INCLUDES) $(APXS_CFLAGS) $(EXTRA_CFLAGS) $(MODSEC_EXTRA_CFLAGS) $(CPPFLAGS) $(APR_CFLAGS) $(APU_CFLAGS) -o msc_test.lo -c msc_test.c diff --git a/apache2/Makefile.win b/apache2/Makefile.win index 1b2c4af3..95a7a3fb 100644 --- a/apache2/Makefile.win +++ b/apache2/Makefile.win @@ -45,7 +45,8 @@ OBJS = mod_security2.obj apache2_config.obj apache2_io.obj apache2_util.obj \ re.obj re_operators.obj re_actions.obj re_tfns.obj re_variables.obj \ msc_logging.obj msc_xml.obj msc_multipart.obj modsecurity.obj \ msc_parsers.obj msc_util.obj msc_pcre.obj persist_dbm.obj \ - msc_reqbody.obj pdf_protect.obj msc_geo.obj acmp.obj msc_lua.obj + msc_reqbody.obj pdf_protect.obj msc_geo.obj acmp.obj msc_lua.obj \ + msc_release.obj all: $(DLL) diff --git a/apache2/configure b/apache2/configure index d8faaaab..4932ebaa 100755 --- a/apache2/configure +++ b/apache2/configure @@ -5831,6 +5831,8 @@ ac_config_files="$ac_config_files Makefile" ac_config_files="$ac_config_files build/apxs-wrapper" if test -e "$PERL"; then + ac_config_files="$ac_config_files mlogc-src/mlogc-batch-load.pl" + ac_config_files="$ac_config_files t/run-unit-tests.pl" ac_config_files="$ac_config_files t/run-regression-tests.pl" @@ -6407,6 +6409,7 @@ do "mod_security2_config.h") CONFIG_HEADERS="$CONFIG_HEADERS mod_security2_config.h" ;; "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;; "build/apxs-wrapper") CONFIG_FILES="$CONFIG_FILES build/apxs-wrapper" ;; + "mlogc-src/mlogc-batch-load.pl") CONFIG_FILES="$CONFIG_FILES mlogc-src/mlogc-batch-load.pl" ;; "t/run-unit-tests.pl") CONFIG_FILES="$CONFIG_FILES t/run-unit-tests.pl" ;; "t/run-regression-tests.pl") CONFIG_FILES="$CONFIG_FILES t/run-regression-tests.pl" ;; "t/gen_rx-pm.pl") CONFIG_FILES="$CONFIG_FILES t/gen_rx-pm.pl" ;; @@ -7022,6 +7025,7 @@ echo "$as_me: $ac_file is unchanged" >&6;} case $ac_file$ac_mode in "build/apxs-wrapper":F) chmod +x build/apxs-wrapper ;; + "mlogc-src/mlogc-batch-load.pl":F) chmod +x mlogc-src/mlogc-batch-load.pl ;; "t/run-unit-tests.pl":F) chmod +x t/run-unit-tests.pl ;; "t/run-regression-tests.pl":F) chmod +x t/run-regression-tests.pl ;; "t/gen_rx-pm.pl":F) chmod +x t/gen_rx-pm.pl ;; diff --git a/apache2/configure.in b/apache2/configure.in index 353e0da9..55c9ff17 100644 --- a/apache2/configure.in +++ b/apache2/configure.in @@ -327,6 +327,7 @@ CHECK_CURL() AC_CONFIG_FILES([Makefile]) AC_CONFIG_FILES([build/apxs-wrapper], [chmod +x build/apxs-wrapper]) if test -e "$PERL"; then + AC_CONFIG_FILES([mlogc-src/mlogc-batch-load.pl], [chmod +x mlogc-src/mlogc-batch-load.pl]) AC_CONFIG_FILES([t/run-unit-tests.pl], [chmod +x t/run-unit-tests.pl]) AC_CONFIG_FILES([t/run-regression-tests.pl], [chmod +x t/run-regression-tests.pl]) AC_CONFIG_FILES([t/gen_rx-pm.pl], [chmod +x t/gen_rx-pm.pl]) diff --git a/apache2/mlogc-src/INSTALL b/apache2/mlogc-src/INSTALL new file mode 100644 index 00000000..095f5a4d --- /dev/null +++ b/apache2/mlogc-src/INSTALL @@ -0,0 +1,76 @@ +ModSecurity Audit Log Collector (mlogc) + +Mlogc is used to connect a ModSecurity sensor to the central +audit log repository. + +To Install: +=========== + + 1) Copy the mlogc executable to an appropriate location. + + A good location might be /usr/local/bin, /opt/mlogc/bin, etc. + + 2) Create sensor in the central audit log repository. Note the + username and the password (SENSOR_USERNAME, SENSOR_PASSWORD). + Also note the IP address central repository listens on + (CONSOLE_IP_ADDRESS). + + 3) Configure the ModSecurity sensor to use mlogc + + # Use ReleventOnly auditing + SecAuditEngine RelevantOnly + + # Must use concurrent logging + SecAuditLogType Concurrent + + # Send all audit log parts + SecAuditLogParts ABIDEFGHZ + + # Use the same /CollectorRoot/LogStorageDir as in mlogc.conf + SecAuditLogStorageDir /var/log/mlogc/data + + # Pipe audit log to mlogc with your configuration + SecAuditLog "|/usr/local/bin/mlogc /etc/mlogc.conf" + + 4) Using the mlogc-default.conf as a template, configure the logger. + + Typically these are the only directives that will need to be modified + to conform to your site: + + # Points to the root of the installation. All relative + # paths configured in this file will be resolved with the + # help of this path (LogStorageDir, TransactionLog, etc.) + # + # Typically, this will be the parent directory that is configured + # in ModSecurity for the SecAuditLogStorageDirectory. So, if + # your SecAuditLogStorageDirectory is set to /var/log/mlogc/data, + # then set this to /var/log/mlogc. + CollectorRoot "/var/log/mlogc" + + # ModSecurity Console receiving URI. You can change the host + # and the port parts but leave everything else as is. + ConsoleURI https://CONSOLE_IP_ADDRESS:8886/rpc/auditLogReceiver + + # Sensor credentials + SensorUsername "SENSOR_USERNAME" + SensorPassword "SENSOR_PASSWORD" + + # Base directory where the audit logs are stored. This can be specified + # as a path relative to the CollectorRoot, or a full path. It should + # resolve to the same path as ModSecurity's SecAuditLogStorageDirectory. + LogStorageDir "data" + + See the mlogc-default.conf configuration file for details on other + configuration directives. + + 5) Restart the ModSecurity sensor. + + From now on every audit log generated will go to the repository. Make + sure you create an alert. Transactions without alerts will be recorded + but not displayed on the home page. + + To troubleshoot, generate alerts and observe file "mlogc-error.log". + + If mlogc fails to connect to the server it will pause for a period + of time (60 seconds by default) before it will try again. + diff --git a/apache2/mlogc-src/Makefile.in b/apache2/mlogc-src/Makefile.in new file mode 100755 index 00000000..9c23aea4 --- /dev/null +++ b/apache2/mlogc-src/Makefile.in @@ -0,0 +1,70 @@ +# Generated Makefile for ModSecurity Log Collector (mlogc) + +CC = @CC@ +EXTRA_CFLAGS = @EXTRA_CFLAGS@ + +srcdir = . +modsecsrcdir = $(srcdir)/.. +srclibdir = $(srcdir)/srclib + +MLOGC_VERSION = `grep '^\#define *VERSION ' mlogc.c | sed 's/.*VERSION *"\([^"]*\)"/\1/'` + +APR_FLAGS = @APR_CFLAGS@ +APR_LIBS = @APR_LINK_LD@ + +CURL_FLAGS = @CURL_CFLAGS@ +CURL_LIBS = @CURL_LIBS@ + +PCRE_FLAGS = @PCRE_CFLAGS@ +PCRE_LIBS = @PCRE_LIBS@ + +APR_S_FLAGS = `$(srclibdir)/install/apr/bin/apr-1-config --includes --cppflags --cflags` +APR_S_LIBS = `$(srclibdir)/install/apr/bin/apr-1-config --link-ld` + +CURL_S_FLAGS = `$(srclibdir)/install/curl/bin/curl-config --cflags` +CURL_S_LIBS = `$(srclibdir)/install/curl/bin/curl-config --libs` + +PCRE_S_FLAGS = `$(srclibdir)/install/pcre/bin/pcre-config --cflags` +PCRE_S_LIBS = `$(srclibdir)/install/pcre/bin/pcre-config --libs` + +all: mlogc + +mlogc: mlogc.c + @echo; \ + echo "Building dynamically linked mlogc..."; \ + $(CC) $(CFLAGS) -o mlogc mlogc.c \ + -I$(modsecsrcdir) \ + $(APR_FLAGS) $(CURL_FLAGS) $(PCRE_FLAGS) \ + $(APR_LIBS) $(CURL_LIBS) $(PCRE_LIBS); \ + chmod 755 mlogc; \ + echo; \ + echo "Build finished. Please follow the INSTALL instructions to complete the install."; \ + echo + +.archives-ok: + @if [ -n "$(MLOGC_NOVERIFY)" -a "$(MLOGC_NOVERIFY)" = "1" ]; then \ + touch .archives-ok; \ + else \ + $(srclibdir)/archives.sh && touch .archives-ok; \ + fi + +.support-libs-ok: + $(srclibdir)/build.sh && touch .support-libs-ok + +archives: .archives-ok + +support-libs: .support-libs-ok + +clean-build: + @rm -rf $(srclibdir)/build + +clean-install: + @rm -rf $(srclibdir)/install + +clean-mlogc: + @rm -rf core mlogc *~ *.o *.so *.lo *.la *.slo + +distclean: clean + +clean: clean-build clean-install clean-mlogc + diff --git a/apache2/mlogc-src/mlogc-batch-load.pl.in b/apache2/mlogc-src/mlogc-batch-load.pl.in new file mode 100755 index 00000000..75d43ba5 --- /dev/null +++ b/apache2/mlogc-src/mlogc-batch-load.pl.in @@ -0,0 +1,151 @@ +#!@PERL@ +# +# ModSecurity for Apache 2.x, http://www.modsecurity.org/ +# Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/) +# +# This product is released under the terms of the General Public Licence, +# version 2 (GPLv2). Please refer to the file LICENSE (included with this +# distribution) which contains the complete text of the licence. +# +# There are special exceptions to the terms and conditions of the GPL +# as it is applied to this software. View the full text of the exception in +# file MODSECURITY_LICENSING_EXCEPTION in the directory of this software +# distribution. +# +# If any of the files related to licensing are missing or if you have any +# other questions related to licensing please contact Breach Security, Inc. +# directly using the email address support@breach.com. +# + +use strict; +use File::Find qw(find); +use File::Spec::Functions qw(catfile); +use Sys::Hostname qw(hostname); +use Digest::MD5 qw(md5_hex); + +my $ROOTDIR = $ARGV[0] || ''; +my $MLOGC = $ARGV[1] || ''; +my $MLOGCCONF = $ARGV[2] || ''; +my @AUDIT = (); + +if ($ROOTDIR eq '' or ! -e $MLOGC or ! -e $MLOGCCONF) { + printf STDERR "\nUsage: $0 \n\n"; + exit 1; +} + +open(MLOGC, "|$MLOGC -f $MLOGCCONF") or die "ERROR: could not open '$MLOGC' - $!\n"; + +find( + { + wanted => sub { + my($fn,$dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size); + + (($dev,$ino,$mode,$nlink,$uid,$gid,$rdev,$size) = stat($_)) && + -f _ && + /^\d{8}-\d+-\w{24}$/s + && (($fn = $File::Find::name) =~ s/^\Q$ROOTDIR\E//) + && push(@AUDIT, [$fn, $size]); + }, + follow => 1, + }, + $ROOTDIR +); + +for my $audit (@AUDIT) { + my $fn = $audit->[0]; + my $line = ""; + my $err = 0; + my $ln = 0; + my $sln = 0; + my $sect = ""; + my $data = ""; + my %data = ( + hostname => hostname(), + remote_addr => "-", + remote_user => "-", + local_user => "-", + logtime => "-", + request => "-", + response_status => "-", + bytes_sent => "-", + referer => "-", + user_agent => "-", + uniqueid => "-", + sessionid => "-", + audit_file => $fn, + extra => "0", + audit_size => $audit->[1], + md5 => "-", + ); + + ### Parse the audit file in an attempt to recreate the original log line + open (AUDIT, "<".catfile($ROOTDIR,$fn)) or $err = 1; + if ($err == 1) { + print STDERR "ERROR: could not open '$fn' - $!\n"; + next; + } + + while($line = ) { + $data .= $line; + chop $line; + $ln++; + $sln++; + if ($line =~ m%^--[0-9A-Fa-f]{8}-([A-Z])--$%) { + $sect = $1; + $sln = 0; + next; + }; + if ($sect eq 'A') { + if ($line =~ m%^(\[[-\d/: a-zA-Z]{27}\]) (\S+) (\S+) (\d+) (\S+) (\d+)%) { + $data{logtime} = $1; + $data{uniqueid} = $2; + $data{remote_addr} = $3; + } + next; + } + elsif ($sect eq 'B') { + if ($sln == 1) { + $data{request} = $line; + } + elsif ($line =~ m%^User=Agent: (.*)%i) { + $data{user_agent} = $1; + } + elsif ($line =~ m%^Referer: (.*)%i) { + $data{referer} = $1; + } + next; + } + elsif ($sect eq 'F') { + if ($sln == 1 and $line =~ m%^\S+ (\d{3})\D?.*%) { + $data{response_status} = $1; + } + elsif ($line =~ m%^Content-Length: (\d+)%i) { + $data{bytes_sent} = $1; + } + next; + } + } + $data{md5} = md5_hex($data); + + printf MLOGC ( + "%s %s %s %s %s \"%s\" %s %s \"%s\" \"%s\" %s \"%s\" %s %s %s md5:%s\n", + $data{hostname}, + $data{remote_addr}, + $data{remote_user}, + $data{local_user}, + $data{logtime}, + $data{request}, + $data{response_status}, + $data{bytes_sent}, + $data{referer}, + $data{user_agent}, + $data{uniqueid}, + $data{sessionid}, + $data{audit_file}, + $data{extra}, + $data{audit_size}, + $data{md5}, + ); + +} + diff --git a/apache2/mlogc-src/mlogc-default.conf b/apache2/mlogc-src/mlogc-default.conf new file mode 100644 index 00000000..96378fef --- /dev/null +++ b/apache2/mlogc-src/mlogc-default.conf @@ -0,0 +1,91 @@ +########################################################################## +# Required configuration +# At a minimum, the items in this section will need to be adjusted to +# fit your environment. The remaining options are optional. +########################################################################## + +# Points to the root of the installation. All relative +# paths will be resolved with the help of this path. +CollectorRoot "/var/log/mlogc" + +# ModSecurity Console receiving URI. You can change the host +# and the port parts but leave everything else as is. +ConsoleURI "https://CONSOLE_IP_ADDRESS:8888/rpc/auditLogReceiver" + +# Sensor credentials +SensorUsername "SENSOR_USERNAME" +SensorPassword "SENSOR_PASSWORD" + +# Base directory where the audit logs are stored. This can be specified +# as a path relative to the CollectorRoot, or a full path. +LogStorageDir "data" + +# Transaction log will contain the information on all log collector +# activities that happen between checkpoints. The transaction log +# is used to recover data in case of a crash (or if Apache kills +# the process). +TransactionLog "mlogc-transaction.log" + +# The file where the pending audit log entry data is kept. This file +# is updated on every checkpoint. +QueuePath "mlogc-queue.log" + +# The location of the error log. +ErrorLog "mlogc-error.log" + +# The location of the lock file. +LockFile "mlogc.lck" + +# Keep audit log entries after sending? (0=false 1=true) +# NOTE: This is required to be set in SecAuditLog mlogc config if you +# are going to use a secondary console via SecAuditLog2. +KeepEntries 0 + + +########################################################################## +# Optional configuration +########################################################################## + +# The error log level controls how much detail there +# will be in the error log. The levels are as follows: +# 0 - NONE +# 1 - ERROR +# 2 - WARNING +# 3 - NOTICE +# 4 - DEBUG +# 5 - DEBUG2 +# +ErrorLogLevel 3 + +# How many concurrent connections to the server +# are we allowed to open at the same time? Log collector uses +# multiple connections in order to speed up audit log transfer. +# This is especially needed when the communication takes place +# over a slow link (e.g. not over a LAN). +MaxConnections 10 + +# The time each connection will sit idle before being reused, +# in milliseconds. Increase if you don't want ModSecurity Console +# to be hit with too many log collector requests. +TransactionDelay 50 + +# The time to wait before initialization on startup in milliseconds. +# Increase if mlogc is starting faster then termination when the +# sensor is reloaded. +StartupDelay 1000 + +# How often is the pending audit log entry data going to be written +# to a file. The default is 15 seconds. +CheckpointInterval 15 + +# If the server fails all threads will back down until the +# problem is sorted. The management thread will periodically +# launch a thread to test the server. The default is to test +# once in 60 seconds. +ServerErrorTimeout 60 + +# The following two parameters are not used yet, but +# reserved for future expansion. +# KeepAlive 150 +# KeepAliveTimeout 300 + diff --git a/apache2/mlogc-src/mlogc.c b/apache2/mlogc-src/mlogc.c new file mode 100644 index 00000000..a36e155a --- /dev/null +++ b/apache2/mlogc-src/mlogc.c @@ -0,0 +1,1865 @@ +/* + * ModSecurity for Apache 2.x, http://www.modsecurity.org/ + * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/) + * + * This product is released under the terms of the General Public Licence, + * version 2 (GPLv2). Please refer to the file LICENSE (included with this + * distribution) which contains the complete text of the licence. + * + * There are special exceptions to the terms and conditions of the GPL + * as it is applied to this software. View the full text of the exception in + * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software + * distribution. + * + * If any of the files related to licensing are missing or if you have any + * other questions related to licensing please contact Breach Security, Inc. + * directly using the email address support@breach.com. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if APR_HAVE_UNISTD_H +#include /* for getpid() */ +#endif +#include +#include +#include +#include +#include + +#include "msc_release.h" + +static void logc_shutdown(int rc); +static void create_new_worker(int lock); +static void error_log(int level, void *thread, const char *text, ...) PRINTF_ATTRIBUTE(3,4); + + +/* -- Constants -- */ + +/* Error log levels. */ +#define LOG_ERROR 1 +#define LOG_WARNING 2 +#define LOG_NOTICE 3 +#define LOG_DEBUG 4 +#define LOG_DEBUG2 5 + +/* The management thread will wake up every five seconds. */ +#define MANAGER_SLEEP 5000000 +#define MANAGER_SUBSLEEP 10000 + +/* Hack to allow multiple mlogc with single delete */ +#define KEEP_ENTRIES_REMOVE_HACK 2600 +#define KEEP_ENTRIES_REMOVE_TIME 0l +#ifdef TEST_HACK +#define TEST_WITH_RAND_SLEEP(n) \ +do { \ + int sec = rand()/(RAND_MAX/n); \ + error_log(LOG_DEBUG2, NULL, "TEST_HACK: Sleeping for %ds", sec); \ + apr_sleep(apr_time_from_sec(sec)); \ +} while(0) +#else +#define TEST_WITH_RAND_SLEEP(n) +#endif + +#define CAPTUREVECTORSIZE 60 +#define PIPE_BUF_SIZE 65536 +#define MEMALLOC_ERROR_MSG "Memory allocation failed!" +#define VERSION MODSEC_VERSION + +#define CMDLINE_OPTS "fh" + +#define IN 0 +#define OUT 1 + +#define STATUSBUF_SIZE 256 + +#define ISHEXCHAR(X) (((X >= '0')&&(X <= '9')) || ((X >= 'a')&&(X <= 'f')) || ((X >= 'A')&&(X <= 'F'))) + +/* -- Regex Patterns -- */ + +/** + * This regular expression is used to parse the entire + * log line we receive from Apache. The REQUEST_LINE is + * treated as a single parameter to allow for invalid + * requests. + */ +const char logline_pattern[] = + "^(\\S+)" + "\\ (\\S+)\\ (\\S+)\\ (\\S+)" + "\\ \\[([^:]+):(\\d+:\\d+:\\d+)\\ ([^\\]]+)\\]" + "\\ \"(.*)\"" + "\\ (\\d+)\\ (\\S+)" + "\\ \"(.*)\"\\ \"(.*)\"" + "\\ (\\S+)\\ \"(.*)\"" + "\\ (\\S+)\\ (\\d+)\\ (\\d+)" + "\\ (\\S+)" + "(.*)$"; + + +/** + * This regular expression can be used to parse + * a REQUEST_LINE field into method, URI, and + * protocol. + */ +const char requestline_pattern[] = + "(\\S+)\\ (.*?)\\ (\\S+)"; + + +/* -- Structures -- */ + +typedef struct { + unsigned long int id; + const char *line; + apr_size_t line_size; +} entry_t; + + +/* -- Global variables -- */ + +pid_t logc_pid = 0; +const char *conffile = NULL; +const char *lockfile = NULL; +int have_read_data = 0; +int checkpoint_interval = 60; +apr_time_t checkpoint_time_last = 0; +const char *collector_root = NULL; +apr_table_t *conf = NULL; +const char *console_uri = NULL; +apr_array_header_t *curl_handles = NULL; +int current_workers = 0; +int management_thread_active = 0; +unsigned long int entry_counter = 1; +const char *error_log_path = NULL; +apr_file_t *error_log_fd = NULL; +int error_log_level = 2; +apr_hash_t *in_progress = NULL; +int keep_alive = 150; /* Not used yet. */ +int keep_alive_timeout = 300; /* Not used yet. */ +int keep_entries = 0; +const char *log_repository = NULL; +void *logline_regex = NULL; +int max_connections = 10; +apr_global_mutex_t *gmutex = NULL; +apr_thread_mutex_t *mutex = NULL; +apr_pool_t *pool = NULL; +apr_array_header_t *queue = NULL; +const char *queue_path = NULL; +/* apr_time_t queue_time = 0; */ +void *requestline_regex = NULL; +int running = 0; +const char *sensor_password = NULL; +const char *sensor_username = NULL; +int server_error = 0; +apr_time_t server_error_last_check_time = 0; +int server_error_timeout = 60; +int startup_delay = 100; +int transaction_delay = 100; +const char *transaction_log_path = NULL; +apr_file_t *transaction_log_fd = NULL; + + +/* -- Commandline opts -- */ +int opt_force = 0; + + +/* -- Code -- */ + +static char *_log_escape(const char *input, apr_size_t input_len) +{ + static const char c2x_table[] = "0123456789abcdef"; + unsigned char *d = NULL; + char *ret = NULL; + unsigned long int i; + + if (input == NULL) return NULL; + + ret = apr_palloc(pool, input_len * 4 + 1); + if (ret == NULL) return NULL; + d = (unsigned char *)ret; + + i = 0; + while(i < input_len) { + switch(input[i]) { + case '"' : + *d++ = '\\'; + *d++ = '"'; + break; + case '\b' : + *d++ = '\\'; + *d++ = 'b'; + break; + case '\n' : + *d++ = '\\'; + *d++ = 'n'; + break; + case '\r' : + *d++ = '\\'; + *d++ = 'r'; + break; + case '\t' : + *d++ = '\\'; + *d++ = 't'; + break; + case '\v' : + *d++ = '\\'; + *d++ = 'v'; + break; + case '\\' : + *d++ = '\\'; + *d++ = '\\'; + break; + default : + if ((input[i] <= 0x1f)||(input[i] >= 0x7f)) { + *d++ = '\\'; + *d++ = 'x'; + *d++ = c2x_table[input[i] >> 4]; + *d++ = c2x_table[input[i] & 0x0f]; + } else { + *d++ = input[i]; + } + break; + } + + i++; + } + + *d = 0; + + return ret; +} + +/** + * Converts a byte given as its hexadecimal representation + * into a proper byte. Handles uppercase and lowercase letters + * but does not check for overflows. + */ +static unsigned char x2c(unsigned char *what) { + register unsigned char digit; + + digit = (what[0] >= 'A' ? ((what[0] & 0xdf) - 'A') + 10 : (what[0] - '0')); + digit *= 16; + digit += (what[1] >= 'A' ? ((what[1] & 0xdf) - 'A') + 10 : (what[1] - '0')); + + return digit; +} + +/** + * URL Decodes a string in-place + */ +static int urldecode_inplace(unsigned char *input, apr_size_t input_len) { + unsigned char *d = (unsigned char *)input; + apr_size_t i; + + if (input == NULL) return 0; + + i = 0; + while (i < input_len) { + if (input[i] == '%') { + /* Character is a percent sign. */ + + /* Are there enough bytes available? */ + if (i + 2 < input_len) { + char c1 = input[i + 1]; + char c2 = input[i + 2]; + + if (ISHEXCHAR(c1) && ISHEXCHAR(c2)) { + /* Valid encoding - decode it. */ + *d++ = x2c(&input[i + 1]); + i += 3; + } else { + /* Not a valid encoding, skip this % */ + *d++ = input[i++]; + } + } else { + /* Not enough bytes available, copy the raw bytes. */ + *d++ = input[i++]; + } + } else { + /* Character is not a percent sign. */ + if (input[i] == '+') { + *d++ = ' '; + } else { + *d++ = input[i]; + } + i++; + } + } + + *d = '\0'; + + return 1; +} + +/** + * Detect a relative path and merge it with the collector root + * path. Leave absolute paths as they are. + */ +static const char *file_path(const char *path) +{ + char *newpath = NULL; + apr_status_t rc; + + if (path == NULL) return NULL; + + rc = apr_filepath_merge(&newpath, collector_root, path, APR_FILEPATH_TRUENAME, pool); + if ((newpath != NULL) && (rc == APR_SUCCESS || APR_STATUS_IS_EPATHWILD(rc) + || APR_STATUS_IS_ENOENT(rc) || APR_STATUS_IS_ENOTDIR(rc))) + { + return newpath; + } + else { + return NULL; + } +} + + +/** + * Returns the current datetime as a string. + */ +static char *current_logtime(char *dest, int dlen) +{ + apr_time_exp_t t; + apr_size_t len; + + apr_time_exp_lt(&t, apr_time_now()); + apr_strftime(dest, &len, dlen, "%a %b %d %H:%M:%S %Y", &t); + + return dest; +} + + +/** + * Logs error to the error log (if available) or + * to the stderr. + */ +static void error_log(int level, void *thread, const char *text, ...) +{ + char msg1[4096] = ""; + char msg2[4096] = ""; + char datetime[100]; + va_list ap; + + if (level > error_log_level) return; + + va_start(ap, text); + + apr_vsnprintf(msg1, sizeof(msg1), text, ap); + apr_snprintf(msg2, sizeof(msg2), "[%s] [%d] [%" APR_PID_T_FMT "/%pp] %s\n", current_logtime(datetime, sizeof(datetime)), level, logc_pid, (thread ? thread : 0), msg1); + + if (error_log_fd != NULL) { + apr_size_t nbytes_written; + apr_size_t nbytes = strlen(msg2); + apr_file_write_full(error_log_fd, msg2, nbytes, &nbytes_written); + } + else { + fprintf(stderr, msg2); + } + + va_end(ap); +} + + +/** + * Adds one entry to the internal queue. It will (optionally) start + * a new thread to handle it. + */ +static void add_entry(const char *data, int start_worker) +{ + entry_t *entry = NULL; + + entry = (entry_t *)malloc(sizeof(entry_t)); + entry->id = 0; + entry->line = strdup(data); + entry->line_size = strlen(entry->line); + + apr_thread_mutex_lock(mutex); + + /* Assign unique ID to this log entry. */ + entry->id = entry_counter++; + + /* Add the new audit log entry to the queue. */ + *(entry_t **)apr_array_push(queue) = entry; + + /* Create a new worker if we can, but not if there is a known problem with the server. */ + if ((start_worker != 0)&&(current_workers < max_connections)&&(server_error == 0)) { + create_new_worker(0); + } + + apr_thread_mutex_unlock(mutex); +} + + +/** + * Read the queue entries. + */ +static int read_queue_entries(apr_file_t *fd, apr_time_t *queue_time) +{ + char linebuf[4100]; + int line_count = -1; + + for(;;) { + apr_status_t rc = apr_file_gets(linebuf, 4096, fd); + char *p; + + if (rc == APR_EOF) break; + if (rc != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Error reading from the queue file."); + logc_shutdown(1); + } + + if (line_count < 0) { + /* First line contains the queue time. */ + *queue_time = (apr_time_t)apr_atoi64(linebuf); + line_count = 0; + continue; + } + + p = &linebuf[0]; + + /* Remove the \n from the end of the line. */ + while(*p != '\0') { + if (*p == '\n') { + *p = '\0'; + break; + } + p++; + } + + if (linebuf[0] == '#') { /* Ignore comments. */ + continue; + } + + add_entry((const char *)&linebuf, 0); + + line_count++; + } + + apr_file_close(fd); + + return line_count; +} + + +/** + * Initialise the transaction log. This code should be + * executed only once at startup. + */ +static void transaction_log_init() +{ + /* ENH: These big enough? */ + char new_queue_path[256]; + char old_queue_path[256]; + apr_file_t *queue_fd = NULL; + apr_time_t queue_time; + + apr_snprintf(new_queue_path, sizeof(new_queue_path), "%s.new", queue_path); + apr_snprintf(old_queue_path, sizeof(old_queue_path), "%s.old", queue_path); + + /* Put a lock in place to ensure exclusivity. */ + if (APR_STATUS_IS_EBUSY(apr_global_mutex_trylock(gmutex))) { + error_log(LOG_WARNING, NULL, "Transaction initialization waiting on mutex"); + } + apr_global_mutex_lock(gmutex); + + error_log(LOG_DEBUG, NULL, "Transaction initialization started."); + + /* Delete .new file if there is one. */ + apr_file_remove(new_queue_path, pool); + + /* Read in the data from the queue. */ + if (apr_file_open(&queue_fd, queue_path, APR_READ | APR_FILE_NOCLEANUP, + 0, pool) == APR_SUCCESS) + { + int line_count = read_queue_entries(queue_fd, &queue_time); + + apr_file_close(queue_fd); + + if (line_count > 0) { + error_log(LOG_NOTICE, NULL, "Loaded %d entries from the queue file.", line_count); + } + } + /* Try the old queue file. */ + else if (apr_file_open(&queue_fd, old_queue_path, APR_READ | APR_FILE_NOCLEANUP, + 0, pool) == APR_SUCCESS) + { + int line_count = read_queue_entries(queue_fd, &queue_time); + apr_file_close(queue_fd); + error_log(LOG_NOTICE, NULL, "Loaded %d entries from the OLD queue file.", line_count); + apr_file_rename(old_queue_path, queue_path, pool); + } + else { + error_log(LOG_NOTICE, NULL, "Queue file not found. New one will be created."); + } + + /* Delete the old queue file. */ + apr_file_remove(old_queue_path, pool); + + checkpoint_time_last = apr_time_now(); + + /* Start fresh with the transaction log. Do note that + * we do not truncate the transaction log on purpose. Apache + * will start copies of piped logging binaries during configuration + * testing. Truncating would erase the log of a currently running + * instance. + */ + if (apr_file_open(&transaction_log_fd, transaction_log_path, APR_WRITE | APR_CREATE + | APR_APPEND | APR_XTHREAD, APR_OS_DEFAULT, pool) != APR_SUCCESS) + { + error_log(LOG_ERROR, NULL, "Failed to open the transaction log: %s\n", transaction_log_path); + apr_global_mutex_unlock(gmutex); + logc_shutdown(1); + } + + /* Unlock */ + apr_global_mutex_unlock(gmutex); + + error_log(LOG_DEBUG, NULL, "Transaction initialization completed."); +} + + +/** + * Log entry event (incoming or outgoing) to the transaction log. + */ +static void transaction_log(int direction, const char *entry) +{ + apr_size_t nbytes, nbytes_written; + char msg[8196] = ""; + + apr_snprintf(msg, sizeof(msg), "%u %s: %s\n", (unsigned int)apr_time_sec(apr_time_now()), + (direction == IN ? "IN" : "OUT"), entry); + nbytes = strlen(msg); + apr_file_write_full(transaction_log_fd, msg, nbytes, &nbytes_written); +} + + +/** + * Executes a checkpoint, which causes the current queue to be + * written to a file and the transaction log to be truncated. + */ +static void transaction_checkpoint() +{ + /* ENH: These big enough? */ + char new_queue_path[256]; + char old_queue_path[256]; + apr_file_t *queue_fd = NULL; + apr_hash_index_t *hi = NULL; + char msg[256]; + int i; + + apr_snprintf(new_queue_path, sizeof(new_queue_path), "%s.new", queue_path); + apr_snprintf(old_queue_path, sizeof(old_queue_path), "%s.old", queue_path); + apr_snprintf(msg, sizeof(msg), "%u\n", (unsigned int)apr_time_sec(apr_time_now())); + + if (! have_read_data) { + error_log(LOG_DEBUG, NULL, "Checkpoint not required."); + return; + } + + /* Put a lock in place to ensure exclusivity. */ + if (APR_STATUS_IS_EBUSY(apr_global_mutex_trylock(gmutex))) { + error_log(LOG_WARNING, NULL, "Checkpoint waiting on mutex"); + } + apr_global_mutex_lock(gmutex); + + error_log(LOG_DEBUG, NULL, "Checkpoint started."); + + /* Dump active entries into a new queue file. */ + if (apr_file_open(&queue_fd, new_queue_path, APR_WRITE | APR_CREATE + | APR_EXCL | APR_TRUNCATE | APR_FILE_NOCLEANUP, APR_OS_DEFAULT, pool) != APR_SUCCESS) + { + error_log(LOG_ERROR, NULL, "Failed to create file: %s", new_queue_path); + apr_global_mutex_unlock(gmutex); + return; + } + + /* Write the time first. */ + apr_file_write_full(queue_fd, msg, strlen(msg), NULL); + + /* Dump the entries sitting in the queue first. */ + for (i = 0; i < queue->nelts; i++) { + entry_t *entry = ((entry_t **)queue->elts)[i]; + apr_file_write_full(queue_fd, entry->line, entry->line_size, NULL); + apr_file_write_full(queue_fd, &"\n", 1, NULL); + } + error_log(LOG_DEBUG2, NULL, "Checkpoint wrote %d queued entries to new queue.", i); + + /* Then dump the ones that are currently being processed. */ + i = 0; + for (hi = apr_hash_first(NULL, in_progress); hi != NULL; hi = apr_hash_next(hi)) { + void *e; + entry_t *entry = NULL; + + i++; + apr_hash_this(hi, NULL, NULL, &e); + entry = e; /* quiet type-punned warning */ + apr_file_write_full(queue_fd, entry->line, entry->line_size, NULL); + apr_file_write_full(queue_fd, &"\n", 1, NULL); + } + error_log(LOG_DEBUG2, NULL, "Checkpoint wrote %d additional entries to new queue.", i); + + apr_file_close(queue_fd); + + /* Switch the files and truncate the transaction log file. */ + apr_file_remove(old_queue_path, pool); + apr_file_rename(queue_path, old_queue_path, pool); + apr_file_rename(new_queue_path, queue_path, pool); + apr_file_remove(old_queue_path, pool); + apr_file_trunc(transaction_log_fd, 0); + + /* Unlock and exit. */ + apr_global_mutex_unlock(gmutex); + + error_log(LOG_DEBUG, NULL, "Checkpoint completed."); +} + + +/** + * Parse one confguration line and add it to the + * configuration table. + */ +static void parse_configuration_line(const char *line, int line_count) +{ + char *start = NULL, *command = NULL; + char *p = NULL; + + /* Remove the trailing newline character. */ + p = (char *)line; + while(*p != '\0') p++; + if ((p > start)&&(*(p - 1) == '\n')) *(p - 1) = '\0'; + + p = (char *)line; + /* Ignore whitespace at the beginning of the line. */ + while(apr_isspace(*p)) p++; + + /* Ignore empty lines and comments. */ + if ((*p == '\0')||(*p == '#')) return; + + start = p; + while(!apr_isspace(*p)&&(*p != '\0')) p++; + + command = apr_pstrmemdup(pool, start, p - start); + + while(apr_isspace(*p)) p++; + + /* Remove whitespace at the end. */ + start = p; + while(*p != '\0') p++; + if (p > start) { + p--; + while(apr_isspace(*p)) { + *p-- = '\0'; + } + } + + /* Remove quotes, but only if we have matching */ + if ((*start == '"') && (p > start) && (*p == '"')) { + start++; + *p-- = '\0'; + } + + /* Take the last directive */ + /* ENH: Error on dup directives? */ + apr_table_set(conf, command, start); +} + + +/** + * Reads configuration from a file. + */ +static void read_configuration() +{ + char linebuf[4096]; + apr_status_t rc; + apr_file_t *fd; + int line_count; + + conf = apr_table_make(pool, 32); + if (conf == NULL) { + error_log(LOG_ERROR, NULL, MEMALLOC_ERROR_MSG); + logc_shutdown(1); + } + + rc = apr_file_open(&fd, conffile, APR_READ | APR_FILE_NOCLEANUP, 0, pool); + if (rc != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Unable to open configuration file: %s", conffile); + logc_shutdown(1); + } + + line_count = 0; + for(;;) { + rc = apr_file_gets(linebuf, 4096, fd); + if (rc == APR_EOF) return; + if (rc != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Error reading from the configuration file."); + logc_shutdown(1); + } + + line_count++; + parse_configuration_line(linebuf, line_count); + } + + apr_file_close(fd); +} + + +/** + * Initialize the configuration. + */ +static void init_configuration() +{ + const char *s = NULL; + + s = apr_table_get(conf, "CollectorRoot"); + if (s != NULL) { + collector_root = s; + } + + s = apr_table_get(conf, "CheckpointInterval"); + if (s != NULL) { + checkpoint_interval = atoi(s); + } + + s = apr_table_get(conf, "ErrorLog"); + if (s != NULL) { + error_log_path = file_path(s); + } + + s = apr_table_get(conf, "ErrorLogLevel"); + if (s != NULL) { + error_log_level = atoi(s); + } + + s = apr_table_get(conf, "QueuePath"); + if (s != NULL) { + queue_path = file_path(s); + } + else { + error_log(LOG_ERROR, NULL, "QueuePath not defined in the configuration file."); + logc_shutdown(1); + } + + s = apr_table_get(conf, "LockFile"); + if (s != NULL) { + lockfile = file_path(s); + } + + s = apr_table_get(conf, "ServerErrorTimeout"); + if (s != NULL) { + server_error_timeout = atoi(s); + } + + s = apr_table_get(conf, "StartupDelay"); + if (s != NULL) { + startup_delay = atoi(s); + } + + s = apr_table_get(conf, "TransactionDelay"); + if (s != NULL) { + transaction_delay = atoi(s); + } + + s = apr_table_get(conf, "TransactionLog"); + if (s != NULL) { + transaction_log_path = file_path(s); + } + + s = apr_table_get(conf, "MaxConnections"); + if (s != NULL) { + int v = atoi(s); + if (v >= 0) max_connections = v; + } + + s = apr_table_get(conf, "KeepAlive"); + if (s != NULL) { + int v = atoi(s); + if (v >= 0) keep_alive = v; + } + + s = apr_table_get(conf, "KeepAliveTimeout"); + if (s != NULL) { + int v = atoi(s); + if (v >= 0) keep_alive_timeout = v; + } + + s = apr_table_get(conf, "LogStorageDir"); + if (s != NULL) { + log_repository = file_path(s); + } + else { + error_log(LOG_ERROR, NULL, "Missing mandatory parameter LogStorageDir.\n"); + logc_shutdown(1); + } + + s = apr_table_get(conf, "ConsoleURI"); + if (s != NULL) { + console_uri = s; + } + else { + error_log(LOG_ERROR, NULL, "Missing mandatory parameter ConsoleURI.\n"); + logc_shutdown(1); + } + + s = apr_table_get(conf, "SensorUsername"); + if (s != NULL) { + sensor_username = s; + } + else { + error_log(LOG_ERROR, NULL, "Missing mandatory parameter SensorUsername.\n"); + logc_shutdown(1); + } + + s = apr_table_get(conf, "SensorPassword"); + if (s != NULL) { + sensor_password = s; + } + else { + error_log(LOG_ERROR, NULL, "Missing mandatory parameter SensorPassword.\n"); + logc_shutdown(1); + } + + s = apr_table_get(conf, "KeepEntries"); + if (s != NULL) { + keep_entries = atoi(s); + } + else { + keep_entries = 0; + } +} + + +/** + * Clean-up resources before process shutdown. + */ +static void logc_cleanup() +{ + curl_global_cleanup(); +} + + +/** + * Shutdown the logger. + */ +static void logc_shutdown(int rc) +{ + /* Tell the threads to shut down. */ + running = 0; + + error_log(LOG_DEBUG, NULL, "Shutting down"); + + /* Wait for the management thread to stop */ + /* ENH: Need a fixed timeout if this never happens */ + while(management_thread_active != 0) { + apr_sleep(10 * 1000); + } + + if (rc == 0) { + error_log(LOG_NOTICE, NULL, "ModSecurity Audit Log Collector %s terminating normally.", VERSION); + } + else { + error_log(LOG_NOTICE, NULL, "ModSecurity Audit Log Collector %s terminating with error %d", VERSION, rc); + } + + if (error_log_fd != NULL) { + apr_file_flush(error_log_fd); + } + + exit(rc); +} + + +/** + * Handle signals. + */ +static int handle_signals(int signum) +{ + switch (signum) { + case SIGHUP: + error_log(LOG_NOTICE, NULL, "Caught SIGHUP, ignored."); + /* ENH: reload config? */ + return 0; + case SIGINT: + error_log(LOG_NOTICE, NULL, "Caught SIGINT, shutting down."); + logc_shutdown(0); + case SIGTERM: + error_log(LOG_NOTICE, NULL, "Caught SIGTERM, shutting down."); + logc_shutdown(0); + case SIGALRM: + error_log(LOG_DEBUG, NULL, "Caught SIGALRM, ignored."); + return 0; + case SIGTSTP: + error_log(LOG_DEBUG, NULL, "Caught SIGTSTP, ignored."); + return 0; + } + + error_log(LOG_NOTICE, NULL, "Caught unexpected signal %d: %s", signum, apr_signal_description_get(signum)); + logc_shutdown(1); + + return 0; /* should never reach */ +} + + +/** + * This function is invoked by Curl to read the response + * body. Since we don't care about the response body the function + * pretends it is retrieving data where it isn't. + */ +size_t curl_writefunction(void *ptr, size_t size, size_t nmemb, void *stream) +{ + unsigned char *data = (unsigned char *)ptr; + unsigned char *status = (unsigned char *)stream; + + /* Grab the status line text from the first line of output */ + if ((status[0] == 0) && (status[1] == 1)) { + apr_size_t i, j; + int ismsg = 0; + + status[1] = 0; /* reset hidden init flag */ + + for (i = 0, j = 0; i < STATUSBUF_SIZE; i++) { + /* We found a line ending so we are done */ + if ( data[i] == '\r' ) { + break; + } + /* Skip to after the first space (where msg is) */ + if (ismsg < 3) { + if ((ismsg == 1) && !isspace(data[i])) { + ismsg++; + } + else if (isspace(data[i])) { + ismsg++; + } + continue; + } + + /* Copy data (msg) from data to status */ + status[j++] = data[i]; + } + status[j] = '\0'; + urldecode_inplace(status, j); + } + + /* do nothing */ + return (size * nmemb); +} + + +/** + * This function is invoked by Curl whenever it has something + * to say. We forward its messages to the error log at level + * DEBUG. + */ +int curl_debugfunction(CURL *curl, curl_infotype infotype, char *data, size_t datalen, void *ourdata) +{ + apr_size_t i, effectivelen; + + if (error_log_level < LOG_DEBUG) return 0; + + effectivelen = datalen; + for(i = 0; i < datalen; i++) { + if ((data[i] == 0x0a)||(data[i] == 0x0d)) { + effectivelen = i; + break; + } + } + + if (infotype == CURLINFO_TEXT) { + error_log(LOG_DEBUG, ourdata, "CURL: %s", _log_escape(data, effectivelen)); + } + + return 0; +} + + +/** + * Initialise the necessary resources and structures. + */ +static void logc_init() +{ + char errstr[1024]; + apr_status_t rc = 0; + const char *errptr = NULL; + int i, erroffset; + + curl_global_init(CURL_GLOBAL_ALL); + atexit(logc_cleanup); + + if ((rc = apr_file_open(&error_log_fd, error_log_path, APR_WRITE | APR_CREATE | APR_APPEND, + APR_OS_DEFAULT, pool)) != APR_SUCCESS) + { + error_log(LOG_ERROR, NULL, "Failed to open the error log %s: %s\n", + error_log_path, apr_strerror(rc, errstr, 1024)); + logc_shutdown(1); + } + + if ( startup_delay > 0 ) { + error_log(LOG_NOTICE, NULL, "ModSecurity Audit Log Collector %s delaying startup for %dms", VERSION, startup_delay); + apr_sleep(startup_delay * 1000); + } + + error_log(LOG_NOTICE, NULL, "ModSecurity Audit Log Collector %s started.", VERSION); + + queue = apr_array_make(pool, 64, sizeof(entry_t *)); + if (queue == NULL) { + error_log(LOG_ERROR, NULL, MEMALLOC_ERROR_MSG); + logc_shutdown(1); + } + + in_progress = apr_hash_make(pool); + if (in_progress == NULL) { + error_log(LOG_ERROR, NULL, MEMALLOC_ERROR_MSG); + logc_shutdown(1); + } + + if ((rc = apr_global_mutex_create(&gmutex, lockfile, APR_LOCK_DEFAULT, pool)) != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Failed to create global mutex: %s", + apr_strerror(rc, errstr, 1024)); + logc_shutdown(1); + } + + if ((rc = apr_thread_mutex_create(&mutex, APR_THREAD_MUTEX_UNNESTED, pool)) != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Failed to create mutex: %s", + apr_strerror(rc, errstr, 1024)); + logc_shutdown(1); + } + + entry_counter = 1; + + curl_handles = apr_array_make(pool, max_connections, sizeof(CURL *)); + if (curl_handles == NULL) { + error_log(LOG_ERROR, NULL, MEMALLOC_ERROR_MSG); + logc_shutdown(1); + } + + /* Initialise a number of Curl handles. */ + for(i = 0; i < max_connections; i++) { + CURL *curl = NULL; + + /* Create cURL handle. */ + curl = curl_easy_init(); + + /* Pre-configure the handle. */ + curl_easy_setopt(curl, CURLOPT_UPLOAD, TRUE); + curl_easy_setopt(curl, CURLOPT_PUT, TRUE); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, NULL); + curl_easy_setopt(curl, CURLOPT_URL, console_uri); + curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT, 15); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, TRUE); + curl_easy_setopt(curl, CURLOPT_HEADER, TRUE); + + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_writefunction); + + *(CURL **)apr_array_push(curl_handles) = curl; + } + + logline_regex = pcre_compile(logline_pattern, PCRE_CASELESS, &errptr, &erroffset, NULL); + if (logline_regex == NULL) { + error_log(LOG_ERROR, NULL, "Failed to compile pattern: %s\n", logline_pattern); + logc_shutdown(1); + } + + requestline_regex = pcre_compile(requestline_pattern, PCRE_CASELESS, &errptr, &erroffset, NULL); + if (requestline_regex == NULL) { + error_log(LOG_ERROR, NULL, "Failed to compile pattern: %s\n", requestline_pattern); + logc_shutdown(1); + } +} + + +/** + * HACK: To allow two mlogcs running against a single dataset we use the + * mtime as a flag for deletion. + * + * 1) Check file date. + * 2) If it is KEEP_ENTRIES_REMOVE_TIME, then remove the file. + * 3) Otherwise set the date and let the other mlogc remove it. + */ +static void keep_entries_hack(apr_pool_t *mp, apr_thread_t *thread, const char *fn) +{ + apr_file_t *f = NULL; + apr_finfo_t finfo; + char errstr[1024]; + apr_status_t rc; + + /* Opening for write as required for exclusive lock */ + if ((rc = apr_file_open(&f, fn, APR_READ|APR_WRITE|APR_APPEND, APR_OS_DEFAULT, mp)) != APR_SUCCESS) { + error_log(LOG_ERROR, thread, "Could not open \"%s\": %s", fn, apr_strerror(rc, errstr, 1024)); + return; + } + + if ((rc = apr_file_lock(f, APR_FLOCK_EXCLUSIVE|APR_FLOCK_NONBLOCK)) != APR_SUCCESS) { + error_log(LOG_DEBUG2, thread, "Waiting for lock on \"%s\": %s", fn, apr_strerror(rc, errstr, 1024)); + if ((rc = apr_file_lock(f, APR_FLOCK_EXCLUSIVE)) != APR_SUCCESS) { + error_log(LOG_ERROR, thread, "Could not lock \"%s\": %s", fn, apr_strerror(rc, errstr, 1024)); + apr_file_close(f); + return; + } + } + error_log(LOG_DEBUG2, thread, "Locked: %s", fn); + + /* For testing only */ + TEST_WITH_RAND_SLEEP(2); + + if ((rc = apr_stat(&finfo, fn, APR_FINFO_MIN, mp)) != APR_SUCCESS) { + error_log(LOG_ERROR, thread, "Could not stat \"%s\": %s", fn, apr_strerror(rc, errstr, 1024)); + error_log(LOG_DEBUG2, thread, "Unlocked: %s", fn); + apr_file_close(f); + return; + } + + if (finfo.mtime != KEEP_ENTRIES_REMOVE_TIME) { + error_log(LOG_DEBUG2, thread, "Set mtime: %s", fn); + if ((rc = apr_file_mtime_set(fn, (apr_time_t)KEEP_ENTRIES_REMOVE_TIME, mp)) != APR_SUCCESS) { + error_log(LOG_ERROR, thread, "Could not set mtime on \"%s\": %s", fn, apr_strerror(rc, errstr, 1024)); + } + error_log(LOG_DEBUG2, thread, "Unlocked: %s", fn); + apr_file_close(f); + return; + } + + + error_log(LOG_DEBUG, thread, "Removing: %s", fn); + error_log(LOG_DEBUG2, thread, "Unlocked: %s", fn); + apr_file_close(f); + apr_file_remove(fn, mp); +} + + +/** + * Worker thread. Works in a loop, fetching jobs from the queue, + * until the queue is empty or it is otherwise told to quit. + */ +static void * APR_THREAD_FUNC thread_worker(apr_thread_t *thread, void *data) +{ + unsigned int loop_count = 0; + CURL *curl = (CURL *)data; + entry_t **entryptr = NULL; + entry_t *entry = NULL; + apr_status_t rc; + apr_finfo_t finfo; + int capturevector[CAPTUREVECTORSIZE]; + int take_new = 1; + apr_pool_t *tpool; + struct curl_slist *headerlist = NULL; + char curl_error_buffer[CURL_ERROR_SIZE] = ""; + + /* There is no need to do the sleep if this was an invalid entry + * as the sleep is just to protect flooding the console server + * with rapid requests. With an invalid entry we never hit the + * server, so we should not delay processing the next event. + */ + int nodelay = 0; + + + /* Each worker uses its own pool to manage memory. To avoid + * memory leaks the pool is cleared after each processed + * entry. + */ + apr_pool_create(&tpool, NULL); + + error_log(LOG_DEBUG, thread, "Worker thread starting."); + + /* Process jobs in a queue until there are no more jobs to process. */ + for(;;) { + nodelay = 0; + + /* Do we need to shut down? */ + if (running == 0) { + error_log(LOG_DEBUG, thread, "We were told to shut down."); + goto THREAD_SHUTDOWN; + } + + /* Is there a problem with the server? We need + * to shut down if there is. Except that we don't + * want to shut down if we were launched to investigate + * if the server came back online (loop_count will be + * zero in that case). + */ + if ((server_error == 1)&&(loop_count != 0)) { + error_log(LOG_DEBUG, thread, "Shutting down due to server error."); + goto THREAD_SHUTDOWN; + } + + loop_count++; + + /* Get a new entry, but only if we need one. */ + if (take_new) { + error_log(LOG_DEBUG, thread, "Locking mutex."); + + apr_thread_mutex_lock(mutex); + + /* Deal with the previous entry. */ + if (entry != NULL) { + error_log(LOG_DEBUG, thread, "Removing previous entry from storage."); + transaction_log(OUT, entry->line); + + /* Remove previous entry from storage. */ + apr_hash_set(in_progress, &entry->id, sizeof(entry->id), NULL); + + /* Release the memory it used to occupy. */ + free((void *)entry->line); + free(entry); + entry = NULL; + } + + error_log(LOG_DEBUG, thread, "Getting one entry from the queue."); + + /* Get one entry. */ + entryptr = (entry_t **)apr_array_pop(queue); + if (entryptr == NULL) { + apr_thread_mutex_unlock(mutex); + error_log(LOG_DEBUG, thread, "No more work for this thread, exiting."); + + goto THREAD_SHUTDOWN; + } + else { + error_log(LOG_DEBUG, thread, "Got one job."); + entry = *entryptr; + apr_hash_set(in_progress, &entry->id, sizeof(entry->id), entry); + } + + apr_thread_mutex_unlock(mutex); + } + + /* Send one entry. */ + + error_log(LOG_DEBUG, thread, "Processing entry."); + take_new = 0; + + rc = pcre_exec(logline_regex, NULL, entry->line, entry->line_size, 0, 0, + capturevector, CAPTUREVECTORSIZE); + if (rc == PCRE_ERROR_NOMATCH) { /* No match. */ + error_log(LOG_WARNING, thread, "Invalid entry (failed to match regex): %s", _log_escape(entry->line, entry->line_size)); + take_new = 1; + nodelay = 1; + } + else if (rc < 0) { /* Error condition. */ + error_log(LOG_WARNING, thread, "Invalid entry (PCRE error %d): %s", rc, _log_escape(entry->line, entry->line_size)); + take_new = 1; + nodelay = 1; + } + else { /* We have a match. */ + char *uniqueid = NULL; + char *auditlogentry = NULL; + char *hash = NULL; + char *summary = NULL; + char *credentials = NULL; + + error_log(LOG_DEBUG, thread, "Regular expression matched."); + + /* For testing only */ + TEST_WITH_RAND_SLEEP(2); + + uniqueid = apr_psprintf(tpool, "%.*s", + (capturevector[2*13+1] - capturevector[2*13]), (entry->line + capturevector[2*13])); + auditlogentry = apr_psprintf(tpool, "%s/%.*s", log_repository, + (capturevector[2*15+1] - capturevector[2*15]), (entry->line + capturevector[2*15])); + hash = apr_psprintf(tpool, "X-Content-Hash: %.*s", + (capturevector[2*18+1] - capturevector[2*15]), (entry->line + capturevector[2*18])); + summary = apr_psprintf(tpool, "X-ForensicLog-Summary: %s", entry->line); + credentials = apr_psprintf(tpool, "%s:%s", sensor_username, sensor_password); + + rc = apr_stat(&finfo, auditlogentry, APR_FINFO_SIZE, tpool); + if (rc == APR_SUCCESS) { + FILE *hd_src; + char response_buf[STATUSBUF_SIZE]; + CURLcode res; + + /* Initialize the respone buffer with a hidden value */ + response_buf[0] = 0; + response_buf[1] = 1; + + error_log(LOG_DEBUG, thread, "File found, activating cURL."); + + curl_easy_setopt(curl, CURLOPT_VERBOSE, 1); + curl_easy_setopt(curl, CURLOPT_DEBUGFUNCTION, curl_debugfunction); + curl_easy_setopt(curl, CURLOPT_DEBUGDATA, thread); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, curl_error_buffer); + curl_easy_setopt(curl, CURLOPT_USERPWD, credentials); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, response_buf); + + headerlist = curl_slist_append(headerlist, "Expect:"); + headerlist = curl_slist_append(headerlist, hash); + headerlist = curl_slist_append(headerlist, summary); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headerlist); + + hd_src = fopen(auditlogentry, "rb"); + if (hd_src == NULL) { + error_log(LOG_WARNING, thread, "Invalid entry (failed to open file for reading): %s", auditlogentry); + take_new = 1; + nodelay = 1; + goto THREAD_CLEANUP; + } + + curl_easy_setopt(curl, CURLOPT_READDATA, hd_src); + curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, finfo.size); + curl_easy_setopt(curl, CURLOPT_INFILESIZE, finfo.size); +#if 0 + mandatory on win32? + curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); +#endif + + res = curl_easy_perform(curl); + + fclose(hd_src); + + if (res == 0) { + long response_code = 0; + + res = curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response_code); + error_log(LOG_DEBUG, thread, "Request returned with status \"%ld %s\": %s", response_code, response_buf, uniqueid); + + + if (response_code == 0) { + /* Assume problem with connection */ + error_log(LOG_WARNING, thread, "Flagging server as errored after failure to retrieve response code for entry %s (cURL code %d): Possible SSL negotiation error", + uniqueid, res); + apr_sleep(1000 * 1000); + take_new = 0; + server_error = 1; + server_error_last_check_time = apr_time_now(); + } + else if (res != 0) { + error_log(LOG_WARNING, thread, "Flagging server as errored after failure to retrieve response code for entry %s (cURL code %d): %s", + uniqueid, res, curl_error_buffer); + apr_sleep(1000 * 1000); + take_new = 0; + server_error = 1; + server_error_last_check_time = apr_time_now(); + } + else { + if (response_code == 200) { + double total_time, upload_size; + + if (server_error == 1) { + error_log(LOG_NOTICE, thread, "Clearing the server error flag after successful entry submission: %s", uniqueid); + } + server_error = 0; + server_error_last_check_time = 0; + + curl_easy_getinfo(curl, CURLINFO_TOTAL_TIME, &total_time); + curl_easy_getinfo(curl, CURLINFO_SIZE_UPLOAD, &upload_size); + + if (!keep_entries) { + error_log(LOG_DEBUG, thread, "Removing: %s", auditlogentry); + apr_file_remove(auditlogentry, tpool); + } + else if (keep_entries == KEEP_ENTRIES_REMOVE_HACK) { + keep_entries_hack(tpool, thread, auditlogentry); + } + + error_log(LOG_NOTICE, thread, "Entry completed (%.3f seconds, %.0f bytes): %s", + total_time, upload_size, + uniqueid); + take_new = 1; + } + else if (response_code == 409) { + /* Assume problem with audit log entry. */ + error_log(LOG_WARNING, thread, "Failed to submit entry with \"409 %s\": %s", + response_buf, uniqueid); + take_new = 1; + } + else { + /* Assume problem with server. */ + error_log(LOG_WARNING, thread, "Flagging server as errored after failure to submit entry %s with HTTP response code %ld: %s", + uniqueid, response_code, response_buf); + server_error = 1; + server_error_last_check_time = apr_time_now(); + take_new = 0; + } + } + } + else { /* Something isn't right. */ + error_log(LOG_WARNING, thread, "Flagging server as errored after failure to submit entry %s (cURL code %d): %s", uniqueid, res, curl_error_buffer); + server_error = 1; + server_error_last_check_time = apr_time_now(); + take_new = 0; + + } + } + else { + error_log(LOG_WARNING, thread, "Invalid entry (file not found %d): %s", rc, auditlogentry); + take_new = 1; + nodelay = 1; + } + } + + THREAD_CLEANUP: + + /* Sleep if we sent data to the server so we do not flood */ + /* ENH: Need to sleep for 1ms in a loop checking for shutdown */ + if ((nodelay == 0) && (transaction_delay > 0)) { + error_log(LOG_DEBUG, thread, "Sleeping for %d msec.", transaction_delay); + apr_sleep(transaction_delay * 1000); + } + + if (headerlist != NULL) { + curl_slist_free_all(headerlist); + headerlist = NULL; + } + + apr_pool_clear(tpool); + + error_log(LOG_DEBUG, thread, "Loop completed."); + } + + THREAD_SHUTDOWN: + + apr_thread_mutex_lock(mutex); + + /* Deal with the previous entry, if any. */ + if (entry != NULL) { + apr_hash_set(in_progress, &entry->id, sizeof(entry->id), NULL); + + if (take_new == 0) { /* Not done. */ + *(entry_t **)apr_array_push(queue) = entry; + } + else { + transaction_log(OUT, entry->line); + free((void *)entry->line); + free(entry); + } + + entry = NULL; + } + + /* Return curl handle to the pool for reuse. */ + *(CURL **)apr_array_push(curl_handles) = curl; + + /* No more work, exit. */ + current_workers--; + + apr_thread_mutex_unlock(mutex); + + apr_pool_destroy(tpool); + + error_log(LOG_DEBUG, thread, "Thread done."); + apr_thread_exit(thread, 0); + + return NULL; +} + + +/** + * Creates one new worker, giving it one of the available + * Curl handles to work with. + */ +static void create_new_worker(int lock) +{ + apr_thread_t *thread = NULL; + CURL **curlptr = NULL; + + if (lock) apr_thread_mutex_lock(mutex); + + /* A sanity check: this part executes under lock and + * we want to make *sure* we don't create more threads + * than we are allowed. + */ + if (current_workers >= max_connections) { + if (lock) apr_thread_mutex_unlock(mutex); + return; + } + + curlptr = (CURL **)apr_array_pop(curl_handles); + if (curlptr != NULL) { + apr_threadattr_t *thread_attrs; + apr_status_t rc; + + apr_threadattr_create(&thread_attrs, pool); + apr_threadattr_detach_set(thread_attrs, 1); + apr_threadattr_stacksize_set(thread_attrs, 1024); + + rc = apr_thread_create(&thread, thread_attrs, thread_worker, *curlptr, pool); + if (rc != APR_SUCCESS) { + apr_thread_mutex_unlock(mutex); + error_log(LOG_ERROR, thread, "Failed to create new worker thread: %d", rc); + logc_shutdown(1); + } + + current_workers++; + } + else { + if (lock) apr_thread_mutex_unlock(mutex); + error_log(LOG_ERROR, thread, "No more cURL handles (Internal Error)."); + logc_shutdown(1); + } + + if (lock) apr_thread_mutex_unlock(mutex); +} + + +/** + * This function implements the management thread. + */ +static void * APR_THREAD_FUNC thread_manager(apr_thread_t *thread, void *data) +{ + apr_time_t last = 0; + apr_time_t now = 0; + + error_log(LOG_DEBUG, thread, "Management thread: Starting."); + + for(;;) { + now = apr_time_now(); + + /* Should we stop running? */ + if (running == 0) { + /* We need to be last */ + error_log(LOG_DEBUG, thread, "Management thread: Waiting for worker threads to finish."); + while(current_workers > 0) { + apr_sleep(10 * 1000); + } + + if (have_read_data) { + error_log(LOG_NOTICE, thread, "Running final transaction checkpoint."); + transaction_checkpoint(); + } + + error_log(LOG_DEBUG, thread, "Management thread: Exiting."); + management_thread_active = 0; + apr_thread_exit(thread, 0); + } + + /* Sleep for a while, but wake up often to check running status */ + if ((last > 0) && ((now - last) < MANAGER_SLEEP)) { + apr_sleep(MANAGER_SUBSLEEP); + continue; + } + last = now; + + error_log(LOG_DEBUG2, thread, "Management thread: Processing"); + + /* When the server is flagged errored we need to + * create a worker thread from time to time to + * investigate. + */ + if (server_error) { + if ((current_workers == 0)&& + (apr_time_sec(now - server_error_last_check_time) > server_error_timeout)) + { + server_error_last_check_time = now; + error_log(LOG_DEBUG, thread, "Management thread: Creating worker thread to investigate server."); + create_new_worker(1); + } + } + else { + if ((current_workers < max_connections)&&(queue->nelts > current_workers)) { + error_log(LOG_DEBUG, thread, "Management thread: Creating worker thread to catch up with the queue."); + create_new_worker(1); + } + } + + /* Initiate a transaction log checkpoint if enough time passed since the last one. */ + if (apr_time_sec(now - checkpoint_time_last) > checkpoint_interval) { + error_log(LOG_DEBUG, thread, "Management thread: Initiating a checkpoint" + " (previous was %" APR_TIME_T_FMT " seconds ago).", apr_time_sec(now - checkpoint_time_last)); + checkpoint_time_last = now; + transaction_checkpoint(); + } + else { + error_log(LOG_DEBUG2, thread, "Management thread: Last checkpoint was %" APR_TIME_T_FMT " seconds ago.", + apr_time_sec(now - checkpoint_time_last)); + } + } + + return NULL; +} + + +/** + * Thread to handle all signals + */ +static void * APR_THREAD_FUNC thread_signals(apr_thread_t *thread, void *data) +{ + apr_status_t rc; + + error_log(LOG_DEBUG, thread, "Signal thread: Starting."); + rc = apr_signal_thread(handle_signals); + if (rc != APR_SUCCESS) { + error_log(LOG_DEBUG, thread, "Signal thread: Error %d", rc); + logc_shutdown(1); + } + + return NULL; +} + + +/** + * The main loop where we receive log entries from + * Apache and add them to the queue, sometimes creating + * new worker threads to handle them. + */ +static void receive_loop() { + apr_file_t *fd_stdin; + apr_size_t nbytes = PIPE_BUF_SIZE; + char *buf = apr_palloc(pool, PIPE_BUF_SIZE + 1); + char errstr[1024]; + apr_size_t evnt = 0; /* Index in buf to first event char */ + apr_size_t curr = 0; /* Index in buf to current processing char */ + apr_size_t next = 0; /* Index in buf to next unused char */ + int done = 0; + int drop_next = 0; + int buffered_events = 0; + + /* Open stdin. */ + if (apr_file_open_stdin(&fd_stdin, pool) != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Unable to open stdin for reading"); + logc_shutdown(1); + } + + /* Always want this NUL terminated */ + buf[PIPE_BUF_SIZE] = '\0'; + + /* Loop forever receiving entries from stdin. */ + while(!done || (curr < next)) { + apr_status_t rc; + + error_log(LOG_DEBUG2, NULL, "Internal state: [evnt \"%" APR_SIZE_T_FMT "\"][curr \"%" APR_SIZE_T_FMT "\"][next \"%" APR_SIZE_T_FMT "\"][nbytes \"%" APR_SIZE_T_FMT "\"]", evnt, curr, next, nbytes); + + /* If we are not done and have the space, read more */ + if (!done && (nbytes > 0)) { + buffered_events = 0; + nbytes = PIPE_BUF_SIZE - next; + rc = apr_file_read(fd_stdin, (buf + next), &nbytes); + if (rc != APR_SUCCESS) { + if (have_read_data) { + error_log(LOG_NOTICE, NULL, "No more data to read, emptying buffer: %s", apr_strerror(rc, errstr, 1024)); + } + done = 1; + } + else { + have_read_data = 1; + if (error_log_level == LOG_DEBUG) { + error_log(LOG_DEBUG, NULL, "Read %" APR_SIZE_T_FMT " bytes from pipe", nbytes); + } + else { + error_log(LOG_DEBUG2, NULL, "Read %" APR_SIZE_T_FMT " bytes from pipe: `%s'", nbytes, _log_escape((buf + next), nbytes)); + } + } + + next += nbytes; + } + + /** + * Each chunk of data we receive can contain one or more lines for + * which we need to find the EOL marker and then queue the event + * up to that. So, find/queue as many lines in the buffer as we + * can. Any remaining data will get shifted back to the beginning + * of the buffer and the buffer size for the next read adjusted. + */ + while(curr < next) { + /* Look for EOL so we can parse the event */ + while((curr < next) && (buf[curr] != 0x0a)) { + curr++; + } + if (buf[curr] == 0x0a) { + buf[curr] = '\0'; + + /* We may have to drop this one if it previously failed */ + if (drop_next) { + error_log(LOG_ERROR, NULL, "Dropping remaining portion of failed event: `%s'", _log_escape((buf + evnt), (curr - evnt))); + drop_next = 0; + } + else { + transaction_log(IN, buf + evnt); + error_log(LOG_DEBUG2, NULL, "Received audit log entry (count %lu queue %d workers %d): %s", + entry_counter, queue->nelts, current_workers, _log_escape((buf + evnt), strlen(buf + evnt))); + add_entry(buf + evnt, 1); + buffered_events++; + } + + /* Advance indexes to next event in buf */ + evnt = curr = curr + 1; + } + else { + error_log(LOG_DEBUG2, NULL, "Event buffer contains partial event: `%s'", _log_escape((buf + evnt), (next - evnt))); + break; + } + } + + + if (buffered_events > 0) { + error_log(LOG_DEBUG, NULL, "Processed %d entries from buffer.", buffered_events); + + /* Move the unused portion of the buffer to the beginning */ + next -= evnt; + curr -= evnt; + memmove(buf, (buf + evnt), next); + + error_log(LOG_DEBUG2, NULL, "Shifted buffer back %" APR_SIZE_T_FMT " and offset %" APR_SIZE_T_FMT " bytes for next read: `%s'", evnt, next, _log_escape(buf, next)); + + evnt = 0; + } + else if (next == PIPE_BUF_SIZE) { + /** + * There is a chance we could fill the buffer, but not have finished + * reading the event (no EOL yet), so we need to say so and drop + * all data until we find the end of the event that is too large. + */ + + if (drop_next) { + error_log(LOG_ERROR, NULL, "Event continuation too large, dropping it as well: `%s'", _log_escape(buf, PIPE_BUF_SIZE)); + } + else { + error_log(LOG_ERROR, NULL, "Event too large, dropping event: `%s'", _log_escape(buf, PIPE_BUF_SIZE)); + } + + /* Rewind buf and mark that we need to drop up to the next event */ + evnt = curr = next = 0; + drop_next = 1; + } + + nbytes = PIPE_BUF_SIZE - next; + } + + /* Wait for queue to empty if specified */ + if ((server_error == 0) && (opt_force != 0) && (queue->nelts > 0)) { + error_log(LOG_NOTICE, NULL, "Waiting for queue to empty (%d active).", queue->nelts); + while ((server_error == 0) && (opt_force != 0) && (queue->nelts > 0)) { + apr_sleep(10 * 1000); + } + if (queue->nelts > 0) { + error_log(LOG_ERROR, NULL, "Could not empty queue (%d active).", queue->nelts); + } + } +} + + +/** + * Creates the management thread. + */ +static void start_management_thread() +{ + apr_thread_t *thread = NULL; + apr_threadattr_t *thread_attrs; + apr_status_t rc; + + apr_threadattr_create(&thread_attrs, pool); + apr_threadattr_detach_set(thread_attrs, 1); + apr_threadattr_stacksize_set(thread_attrs, 1024); + + management_thread_active = 1; + + rc = apr_thread_create(&thread, thread_attrs, thread_manager, NULL, pool); + if (rc != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Failed to create new management thread: %d", rc); + management_thread_active = 0; + logc_shutdown(1); + } +} + +/** + * Creates a thread to handle all signals + */ +static void start_signal_thread() +{ + apr_thread_t *thread = NULL; + apr_threadattr_t *thread_attrs; + apr_status_t rc; + + apr_threadattr_create(&thread_attrs, pool); + apr_threadattr_detach_set(thread_attrs, 1); + apr_threadattr_stacksize_set(thread_attrs, 1024); + + rc = apr_thread_create(&thread, thread_attrs, thread_signals, NULL, pool); + if (rc != APR_SUCCESS) { + error_log(LOG_ERROR, NULL, "Failed to create new signal thread: %d", rc); + logc_shutdown(1); + } +} + +/** + * Usage text. + */ +static void usage() { + fprintf(stderr, "ModSecurity Log Collector v%s\n", VERSION); + fprintf(stderr, " Usage: mlogc [options] /path/to/the/configuration.file\n"); + fprintf(stderr, "\n"); + fprintf(stderr, " Options:\n"); + fprintf(stderr, " -f Force depletion of queue on exit\n"); + fprintf(stderr, " -h This help\n\n"); +} + + +/** + * This is the main entry point. + */ +int main(int argc, const char * const argv[]) { + apr_getopt_t *opt; + apr_status_t rc; + + apr_app_initialize(&argc, &argv, NULL); + atexit(apr_terminate); + + logc_pid = getpid(); + apr_pool_create(&pool, NULL); + apr_setup_signal_thread(); + + if (argc < 2) { + usage(); + logc_shutdown(1); + } + + /* Commandline opts */ + rc = apr_getopt_init(&opt, pool, argc, argv); + if (rc != APR_SUCCESS) { + usage(); + logc_shutdown(1); + } + + do { + char ch; + const char *val; + rc = apr_getopt(opt, CMDLINE_OPTS, &ch, &val); + switch (rc) { + case APR_SUCCESS: + switch (ch) { + case 'f': + opt_force = 1; + break; + case 'h': + usage(); + logc_shutdown(0); + } + break; + case APR_BADCH: + case APR_BADARG: + usage(); + logc_shutdown(1); + } + } while (rc != APR_EOF); + + /* Conf file is last */ + conffile = argv[argc - 1]; + + read_configuration(); + init_configuration(); + + logc_init(); + transaction_log_init(); + + running = 1; + server_error = 0; + + start_management_thread(); + start_signal_thread(); + + /* Process stdin until EOF */ + receive_loop(); + + logc_shutdown(0); + + return 0; +} diff --git a/apache2/mod_security2.c b/apache2/mod_security2.c index 3991c072..cb2f2ac5 100644 --- a/apache2/mod_security2.c +++ b/apache2/mod_security2.c @@ -25,6 +25,7 @@ #include "apache2.h" #include "http_main.h" #include "pdf_protect.h" + #include "msc_logging.h" #include "msc_util.h" @@ -520,7 +521,7 @@ static int hook_post_config(apr_pool_t *mp, apr_pool_t *mp_log, apr_pool_t *mp_t /* Log our presence to the error log. */ if (first_time) { ap_log_error(APLOG_MARK, APLOG_NOTICE | APLOG_NOERRNO, 0, s, - "%s configured.", MODULE_NAME_FULL); + "%s configured.", MODSEC_MODULE_NAME_FULL); /* If we've changed the server signature make note of the original. */ if (new_server_signature != NULL) { diff --git a/apache2/modsecurity.c b/apache2/modsecurity.c index fd8236ea..071ccd23 100644 --- a/apache2/modsecurity.c +++ b/apache2/modsecurity.c @@ -25,15 +25,6 @@ #include "msc_util.h" #include "msc_xml.h" -modsec_build_type_rec DSOLOCAL modsec_build_type[] = { - { "-dev", 1 }, /* Development build */ - { "-rc", 3 }, /* Release Candidate build */ - { "", 9 }, /* Production build */ - { "-breach", 9 }, /* Breach build */ - { "-trunk", 9 }, /* Trunk build */ - { NULL, -1 } /* terminator */ -}; - /** * Log an alert message to the log, adding the rule metadata at the end. */ diff --git a/apache2/modsecurity.h b/apache2/modsecurity.h index 5d840df6..a745db69 100644 --- a/apache2/modsecurity.h +++ b/apache2/modsecurity.h @@ -32,23 +32,7 @@ typedef struct msc_data_chunk msc_data_chunk; typedef struct msc_arg msc_arg; typedef struct msc_string msc_string; -#if !(defined(WIN32) || defined(CYGWIN) || defined(NETWARE) || defined(SOLARIS2)) -#define DSOLOCAL __attribute__((visibility("hidden"))) -#else -#define DSOLOCAL -#endif - -#if defined(DEBUG_MEM) -/* Nothing Yet */ -#endif - -/* For GNU C, tell the compiler to check printf like formatters */ -#if (defined(__GNUC__) && !defined(SOLARIS2)) -#define PRINTF_ATTRIBUTE(a,b) __attribute__((format (printf, a, b))) -#else -#define PRINTF_ATTRIBUTE(a,b) -#endif - +#include "msc_release.h" #include "msc_logging.h" #include "msc_multipart.h" #include "msc_pcre.h" @@ -66,27 +50,6 @@ typedef struct msc_string msc_string; #include "http_log.h" #include "http_protocol.h" -typedef struct modsec_build_type_rec { - const char * name; - int val; -} modsec_build_type_rec; -extern DSOLOCAL modsec_build_type_rec modsec_build_type[]; - -#define MODSEC_VERSION_MAJOR "2" -#define MODSEC_VERSION_MINOR "6" -#define MODSEC_VERSION_MAINT "0" -#define MODSEC_VERSION_TYPE "-trunk" -#define MODSEC_VERSION_RELEASE "" - -#define MODULE_NAME "ModSecurity for Apache" - -#define MODSEC_VERSION_SUFFIX MODSEC_VERSION_TYPE MODSEC_VERSION_RELEASE -#define MODULE_RELEASE \ - MODSEC_VERSION_MAJOR "." MODSEC_VERSION_MINOR "." MODSEC_VERSION_MAINT \ - MODSEC_VERSION_SUFFIX - -#define MODULE_NAME_FULL MODULE_NAME "/" MODULE_RELEASE " (http://www.modsecurity.org/)" - #define PHASE_REQUEST_HEADERS 1 #define PHASE_REQUEST_BODY 2 #define PHASE_RESPONSE_HEADERS 3 diff --git a/apache2/msc_logging.c b/apache2/msc_logging.c index 68435870..eb6ee5fa 100644 --- a/apache2/msc_logging.c +++ b/apache2/msc_logging.c @@ -343,14 +343,14 @@ static void sec_auditlog_write_producer_header(modsec_rec *msr) { /* Try to write everything in one go. */ if (msr->txcfg->component_signatures->nelts == 0) { - text = apr_psprintf(msr->mp, "Producer: %s.\n", MODULE_NAME_FULL); + text = apr_psprintf(msr->mp, "Producer: %s.\n", MODSEC_MODULE_NAME_FULL); sec_auditlog_write(msr, text, strlen(text)); return; } /* Start with the ModSecurity signature. */ - text = apr_psprintf(msr->mp, "Producer: %s", MODULE_NAME_FULL); + text = apr_psprintf(msr->mp, "Producer: %s", MODSEC_MODULE_NAME_FULL); sec_auditlog_write(msr, text, strlen(text)); diff --git a/apache2/msc_release.c b/apache2/msc_release.c new file mode 100644 index 00000000..462131e3 --- /dev/null +++ b/apache2/msc_release.c @@ -0,0 +1,42 @@ +/* + * ModSecurity for Apache 2.x, http://www.modsecurity.org/ + * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/) + * + * This product is released under the terms of the General Public Licence, + * version 2 (GPLv2). Please refer to the file LICENSE (included with this + * distribution) which contains the complete text of the licence. + * + * There are special exceptions to the terms and conditions of the GPL + * as it is applied to this software. View the full text of the exception in + * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software + * distribution. + * + * If any of the files related to licensing are missing or if you have any + * other questions related to licensing please contact Breach Security, Inc. + * directly using the email address support@breach.com. + * + */ + +#include "msc_release.h" + +modsec_build_type_rec modsec_build_type[] = { + { "-dev", 1 }, /* Development build */ + { "-rc", 3 }, /* Release Candidate build */ + { "", 9 }, /* Production build */ + { "-breach", 9 }, /* Breach build */ + { "-trunk", 9 }, /* Trunk build */ + { NULL, -1 } /* terminator */ +}; + +int get_modsec_build_type(const char *name) +{ + int i; + + for (i = 0; modsec_build_type[i].name != NULL; i++) { + if (strcmp(((name == NULL) ? MODSEC_VERSION_TYPE : name), modsec_build_type[i].name) == 0) { + return modsec_build_type[i].val; + } + } + + return 9; /* so no warning */ +} diff --git a/apache2/msc_release.h b/apache2/msc_release.h new file mode 100644 index 00000000..284ec954 --- /dev/null +++ b/apache2/msc_release.h @@ -0,0 +1,67 @@ +/* + * ModSecurity for Apache 2.x, http://www.modsecurity.org/ + * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/) + * + * This product is released under the terms of the General Public Licence, + * version 2 (GPLv2). Please refer to the file LICENSE (included with this + * distribution) which contains the complete text of the licence. + * + * There are special exceptions to the terms and conditions of the GPL + * as it is applied to this software. View the full text of the exception in + * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software + * distribution. + * + * If any of the files related to licensing are missing or if you have any + * other questions related to licensing please contact Breach Security, Inc. + * directly using the email address support@breach.com. + * + */ +#ifndef _MSC_RELEASE_H_ +#define _MSC_RELEASE_H_ + +#include +#include + +#if !(defined(WIN32) || defined(CYGWIN) || defined(NETWARE) || defined(SOLARIS2)) +#define DSOLOCAL __attribute__((visibility("hidden"))) +#else +#define DSOLOCAL +#endif + +#if defined(DEBUG_MEM) +/* Nothing Yet */ +#endif + +/* For GNU C, tell the compiler to check printf like formatters */ +#if (defined(__GNUC__) && !defined(SOLARIS2)) +#define PRINTF_ATTRIBUTE(a,b) __attribute__((format (printf, a, b))) +#else +#define PRINTF_ATTRIBUTE(a,b) +#endif + +typedef struct modsec_build_type_rec { + const char * name; + int val; +} modsec_build_type_rec; +extern DSOLOCAL modsec_build_type_rec modsec_build_type[]; + +#define MODSEC_VERSION_MAJOR "2" +#define MODSEC_VERSION_MINOR "6" +#define MODSEC_VERSION_MAINT "0" +#define MODSEC_VERSION_TYPE "-trunk" +#define MODSEC_VERSION_RELEASE "" + +#define MODSEC_VERSION_SUFFIX MODSEC_VERSION_TYPE MODSEC_VERSION_RELEASE + +#define MODSEC_VERSION \ + MODSEC_VERSION_MAJOR "." MODSEC_VERSION_MINOR "." MODSEC_VERSION_MAINT \ + MODSEC_VERSION_SUFFIX + +/* Apache Module Defines */ +#define MODSEC_MODULE_NAME "ModSecurity for Apache" +#define MODSEC_MODULE_VERSION MODSEC_VERSION +#define MODSEC_MODULE_NAME_FULL MODSEC_MODULE_NAME "/" MODSEC_MODULE_VERSION " (http://www.modsecurity.org/)" + +int DSOLOCAL get_modsec_build_type(const char *name); + +#endif /* _MSC_RELEASE_H_ */ diff --git a/apache2/msc_test.c b/apache2/msc_test.c index 02866025..e526d650 100644 --- a/apache2/msc_test.c +++ b/apache2/msc_test.c @@ -533,7 +533,7 @@ static void init_msr() { * Usage text. */ static void usage() { - fprintf(stderr, "ModSecurity Unit Tester v%s\n", MODULE_RELEASE); + fprintf(stderr, "ModSecurity Unit Tester v%s\n", MODSEC_VERSION); fprintf(stderr, " Usage: msc_test [options]\n"); fprintf(stderr, "\n"); fprintf(stderr, " Options:\n"); diff --git a/apache2/msc_util.c b/apache2/msc_util.c index 3bb754a3..4816c4d7 100644 --- a/apache2/msc_util.c +++ b/apache2/msc_util.c @@ -16,6 +16,7 @@ * directly using the email address support@breach.com. * */ +#include "msc_release.h" #include "msc_util.h" #include @@ -1169,21 +1170,11 @@ int normalise_path_inplace(unsigned char *input, int input_len, int win, int *ch } char *modsec_build(apr_pool_t *mp) { - int build_type = 0; - int i; - - for (i = 0; modsec_build_type[i].name != NULL; i++) { - if (strcmp(MODSEC_VERSION_TYPE, modsec_build_type[i].name) == 0) { - build_type = modsec_build_type[i].val; - break; - } - } - return apr_psprintf(mp, "%02i%02i%02i%1i%02i", atoi(MODSEC_VERSION_MAJOR), atoi(MODSEC_VERSION_MINOR), atoi(MODSEC_VERSION_MAINT), - build_type, + get_modsec_build_type(NULL), atoi(MODSEC_VERSION_RELEASE)); }