Refactoring on the audit logs implementation

Among of other things, it is now supporting shared file locks between
different process.
This commit is contained in:
Felipe Zimmerle
2016-12-14 10:09:53 -03:00
parent 9707d46e45
commit 2e9a35c358
24 changed files with 661 additions and 260 deletions

View File

@@ -13,7 +13,7 @@
*
*/
#include "src/audit_log/audit_log.h"
#include "modsecurity/audit_log.h"
#include <stddef.h>
#include <stdio.h>
@@ -24,6 +24,7 @@
#include "src/audit_log/writer/https.h"
#include "src/audit_log/writer/parallel.h"
#include "src/audit_log/writer/serial.h"
#include "src/audit_log/writer/writer.h"
#include "src/utils/regex.h"
#define PARTS_CONSTAINS(a, c) \
@@ -38,40 +39,37 @@
parts = parts & ~c; \
}
#define AL_MERGE_STRING_CONF(a, c) \
if (a.empty() == false) { \
c = a; \
}
namespace modsecurity {
namespace audit_log {
AuditLog::AuditLog()
: m_path1(""),
m_path2(""),
m_storage_dir(""),
m_filePermission(0600),
m_directoryPermission(0766),
m_parts(AAuditLogPart | BAuditLogPart | CAuditLogPart | FAuditLogPart
| HAuditLogPart | ZAuditLogPart),
m_status(OffAuditLogStatus),
m_type(ParallelAuditLogType),
m_filePermission(-1),
m_directoryPermission(-1),
m_parts(-1),
m_status(NotSetLogStatus),
m_type(NotSetAuditLogType),
m_relevant(""),
m_writer(NULL),
m_refereceCount(0) { }
m_refereceCount(1) { }
AuditLog::~AuditLog() {
if (m_writer) {
m_writer->refCountDecreaseAndCheck();
delete m_writer;
m_writer = NULL;
}
}
void AuditLog::refCountIncrease() {
m_refereceCount++;
}
void AuditLog::refCountDecreaseAndCheck() {
m_refereceCount--;
if (m_refereceCount == 0) {
delete this;
}
}
bool AuditLog::setStorageDirMode(int permission) {
this->m_directoryPermission = permission;
@@ -85,8 +83,24 @@ bool AuditLog::setFileMode(int permission) {
}
bool AuditLog::setStatus(AuditLogStatus new_status) {
this->m_status = new_status;
int AuditLog::getFilePermission() {
if (m_filePermission == -1) {
return m_defaultFilePermission;
}
return m_filePermission;
}
int AuditLog::getDirectoryPermission() {
if (m_directoryPermission == -1) {
return m_defaultDirectoryPermission;
}
return m_directoryPermission;
}
bool AuditLog::setStatus(AuditLogStatus status) {
this->m_status = status;
return true;
}
@@ -172,34 +186,46 @@ bool AuditLog::setParts(const std::basic_string<char>& new_parts) {
}
int AuditLog::getParts() {
if (m_parts == -1) {
return m_defaultParts;
}
return m_parts;
}
bool AuditLog::setType(AuditLogType audit_type) {
this->m_type = audit_type;
return true;
}
bool AuditLog::init() {
if (m_type == ParallelAuditLogType) {
m_writer = new audit_log::writer::Parallel(this);
}
bool AuditLog::init(std::string *error) {
if (m_type == SerialAuditLogType) {
m_writer = new audit_log::writer::Serial(this);
}
if (m_type == HttpsAuditLogType) {
} else if (m_type == HttpsAuditLogType) {
m_writer = new audit_log::writer::Https(this);
} else {
/*
* if (m_type == ParallelAuditLogType
* || m_type == NotSetAuditLogType)
*
*/
m_writer = new audit_log::writer::Parallel(this);
}
m_writer->refCountIncrease();
if (m_writer == NULL || m_writer->init() == false) {
std::cout << "not able to open the log for write." << std::endl;
if (m_writer == NULL || m_writer->init(error) == false) {
return false;
}
/* Sanity check */
if (m_status == RelevantOnlyAuditLogStatus) {
if (m_relevant.empty()) {
std::cout << "m_relevant cannot be null while status is " << \
"RelevantOnly" << std::endl;
error->assign("m_relevant cannot be null while status is set to " \
"RelevantOnly");
return false;
}
}
@@ -256,7 +282,16 @@ bool AuditLog::saveIfRelevant(Transaction *transaction, int parts) {
}
transaction->debug(5, "Saving this request as part " \
"of the audit logs.");
m_writer->write(transaction, parts);
if (m_writer == NULL) {
transaction->debug(1, "Internal error, audit log writer is null");
} else {
std::string error;
bool a = m_writer->write(transaction, parts, &error);
if (a == false) {
transaction->debug(1, "Cannot save the audit log: " + error);
return false;
}
}
return true;
}
@@ -267,5 +302,35 @@ bool AuditLog::close() {
}
bool AuditLog::merge(AuditLog *from, std::string *error) {
AL_MERGE_STRING_CONF(from->m_path1, m_path1);
AL_MERGE_STRING_CONF(from->m_path2, m_path2);
AL_MERGE_STRING_CONF(from->m_storage_dir, m_storage_dir);
AL_MERGE_STRING_CONF(from->m_relevant, m_relevant);
if (from->m_filePermission != -1) {
m_filePermission = from->m_filePermission;
}
if (from->m_directoryPermission != -1) {
m_directoryPermission = from->m_directoryPermission;
}
if (from->m_type != NotSetAuditLogType) {
m_type = from->m_type;
}
if (from->m_status != NotSetLogStatus) {
m_status = from->m_status;
}
if (from->m_parts != -1) {
m_parts = from->m_parts;
}
return init(error);
}
} // namespace audit_log
} // namespace modsecurity

View File

@@ -1,189 +0,0 @@
/*
* ModSecurity, http://www.modsecurity.org/
* Copyright (c) 2015 Trustwave Holdings, Inc. (http://www.trustwave.com/)
*
* You may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* If any of the files related to licensing are missing or if you have any
* other questions related to licensing please contact Trustwave Holdings, Inc.
* directly using the email address security@modsecurity.org.
*
*/
#ifdef __cplusplus
#include <iostream>
#include <fstream>
#include <string>
#endif
#ifndef SRC_AUDIT_LOG_AUDIT_LOG_H_
#define SRC_AUDIT_LOG_AUDIT_LOG_H_
#include "modsecurity/transaction.h"
#include "src/audit_log/writer.h"
#ifdef __cplusplus
namespace modsecurity {
namespace audit_log {
/** @ingroup ModSecurity_CPP_API */
class AuditLog {
public:
AuditLog();
~AuditLog();
void refCountIncrease();
void refCountDecreaseAndCheck();
enum AuditLogType {
SerialAuditLogType,
ParallelAuditLogType,
HttpsAuditLogType
};
enum AuditLogStatus {
OnAuditLogStatus,
OffAuditLogStatus,
RelevantOnlyAuditLogStatus
};
enum AuditLogParts {
/**
* Audit log header (mandatory).
*
*/
AAuditLogPart = 2,
/**
* Request headers.
*
*/
BAuditLogPart = 4,
/**
* Request body (present only if the request body exists and ModSecurity
* is configured to intercept it).
*
*/
CAuditLogPart = 8,
/**
* Reserved for intermediary response headers; not implemented yet.
*
*/
DAuditLogPart = 16,
/**
* Intermediary response body (present only if ModSecurity is configured
* to intercept response bodies, and if the audit log engine is
* configured to record it). Intermediary response body is the same as the
* actual response body unless ModSecurity intercepts the intermediary
* response body, in which case the actual response body will contain the
* error message (either the Apache default error message, or the
* ErrorDocument page).
*
*/
EAuditLogPart = 32,
/**
* Final response headers (excluding the Date and Server headers, which
* are always added by Apache in the late stage of content delivery).
*
*/
FAuditLogPart = 64,
/**
* Reserved for the actual response body; not implemented yet.
*
*/
GAuditLogPart = 128,
/**
* Audit log trailer.
*
*/
HAuditLogPart = 256,
/**
* This part is a replacement for part C. It will log the same data as C
* in all cases except when multipart/form-data encoding in used. In this
* case, it will log a fake application/x-www-form-urlencoded body that
* contains the information about parameters but not about the files. This
* is handy if you dont want to have (often large) files stored in your
* audit logs.
*
*/
IAuditLogPart = 512,
/**
* This part contains information about the files uploaded using
* multipart/form-data encoding.
*/
JAuditLogPart = 1024,
/**
* This part contains a full list of every rule that matched (one per
* line) in the order they were matched. The rules are fully qualified and
* will thus show inherited actions and default operators. Supported as of
* v2.5.0.
*
*/
KAuditLogPart = 2048,
/**
* Final boundary, signifies the end of the entry (mandatory).
*
*/
ZAuditLogPart = 4096
};
bool setStorageDirMode(int permission);
bool setFileMode(int permission);
bool setStatus(AuditLogStatus new_status);
bool setRelevantStatus(const std::basic_string<char>& new_relevant_status);
bool setFilePath1(const std::basic_string<char>& path);
bool setFilePath2(const std::basic_string<char>& path);
bool setStorageDir(const std::basic_string<char>& path);
bool setParts(const std::basic_string<char>& new_parts);
bool setType(AuditLogType audit_type);
bool init();
bool close();
bool saveIfRelevant(Transaction *transaction);
bool saveIfRelevant(Transaction *transaction, int parts);
bool isRelevant(int status);
int addParts(int parts, const std::string& new_parts);
int removeParts(int parts, const std::string& new_parts);
std::string m_path1;
std::string m_path2;
std::string m_storage_dir;
int m_filePermission;
int m_directoryPermission;
int m_parts;
private:
AuditLogStatus m_status;
AuditLogType m_type;
std::string m_relevant;
audit_log::Writer *m_writer;
int m_refereceCount;
};
} // namespace audit_log
} // namespace modsecurity
#endif
#endif // SRC_AUDIT_LOG_AUDIT_LOG_H_

View File

@@ -25,7 +25,7 @@
#include <fstream>
#include <mutex>
#include "src/audit_log/audit_log.h"
#include "modsecurity/audit_log.h"
#include "modsecurity/transaction.h"
#include "src/utils/md5.h"
#include "src/utils/https_client.h"
@@ -40,12 +40,12 @@ Https::~Https() {
}
bool Https::init() {
bool Https::init(std::string *error) {
return true;
}
bool Https::write(Transaction *transaction, int parts) {
bool Https::write(Transaction *transaction, int parts, std::string *error) {
Utils::HttpsClient m_http_client;
transaction->debug(7, "Sending logs to: " + m_audit->m_path1);

View File

@@ -22,7 +22,7 @@
#ifndef SRC_AUDIT_LOG_WRITER_HTTPS_H_
#define SRC_AUDIT_LOG_WRITER_HTTPS_H_
#include "src/audit_log/writer.h"
#include "src/audit_log/writer/writer.h"
#include "modsecurity/transaction.h"
#ifdef __cplusplus
@@ -32,27 +32,16 @@ namespace audit_log {
namespace writer {
/** @ingroup ModSecurity_CPP_API */
class Https : public audit_log::Writer {
class Https : public Writer {
public:
explicit Https(audit_log::AuditLog *audit)
: audit_log::Writer(audit) { }
: audit_log::writer::Writer(audit) { }
~Https() override;
void refCountIncrease() override {
m_refereceCount++;
}
void refCountDecreaseAndCheck() override {
m_refereceCount--;
if (m_refereceCount == 0) {
delete this;
}
}
bool init() override;
bool write(Transaction *transaction, int parts) override;
bool init(std::string *error) override;
bool write(Transaction *transaction, int parts,
std::string *error) override;
};
} // namespace writer

View File

@@ -25,7 +25,7 @@
#include <fstream>
#include <mutex>
#include "src/audit_log/audit_log.h"
#include "modsecurity/audit_log.h"
#include "modsecurity/transaction.h"
#include "src/utils/system.h"
#include "src/utils/md5.h"
@@ -36,17 +36,9 @@ namespace audit_log {
namespace writer {
std::mutex g_writeMutex;
Parallel::~Parallel() {
if (log1.is_open()) {
log1.close();
}
if (log2.is_open()) {
log2.close();
}
utils::SharedFiles::getInstance().close(m_audit->m_path1);
utils::SharedFiles::getInstance().close(m_audit->m_path2);
}
@@ -80,68 +72,106 @@ inline std::string Parallel::logFilePath(time_t *t,
}
bool Parallel::init() {
/** TODO:: Check if the directory exists. */
/** TODO:: Checking if we have permission to write in the target dir */
bool Parallel::init(std::string *error) {
bool ret = true;
if (!m_audit->m_path1.empty()) {
log1.open(m_audit->m_path1, std::fstream::out | std::fstream::app);
ret = utils::SharedFiles::getInstance().open(m_audit->m_path1, error);
if (!ret) {
return false;
}
}
if (!m_audit->m_path2.empty()) {
log2.open(m_audit->m_path2, std::fstream::out | std::fstream::app);
ret = utils::SharedFiles::getInstance().open(m_audit->m_path2, error);
if (!ret) {
return false;
}
}
if (m_audit->m_storage_dir.empty() == false) {
if (utils::createDir(m_audit->m_storage_dir,
m_audit->getDirectoryPermission(), error) == false) {
return false;
}
}
return true;
}
bool Parallel::write(Transaction *transaction, int parts) {
std::lock_guard<std::mutex> guard(g_writeMutex);
bool Parallel::write(Transaction *transaction, int parts, std::string *error) {
FILE *fp;
int fd;
std::string log = transaction->toJSON(parts);
std::string fileName = logFilePath(&transaction->m_timeStamp,
YearMonthDayDirectory | YearMonthDayAndTimeDirectory
| YearMonthDayAndTimeFileName);
bool ret;
std::string logPath = m_audit->m_storage_dir;
fileName = logPath + fileName + "-" + transaction->m_id;
if (logPath.empty()) {
return false;
error->assign("Log path is not valid.");
return false;
}
utils::createDir((logPath +
ret = utils::createDir((logPath +
logFilePath(&transaction->m_timeStamp, YearMonthDayDirectory)).c_str(),
m_audit->m_directoryPermission);
utils::createDir((logPath +
m_audit->getDirectoryPermission(),
error);
if (ret == false) {
return false;
}
ret = utils::createDir((logPath +
logFilePath(&transaction->m_timeStamp, YearMonthDayDirectory
| YearMonthDayAndTimeDirectory)).c_str(),
m_audit->m_directoryPermission);
m_audit->getDirectoryPermission(),
error);
if (ret == false) {
return false;
}
fd = open(fileName.c_str(), O_CREAT | O_WRONLY, m_audit->m_filePermission);
fd = open(fileName.c_str(), O_CREAT | O_WRONLY,
m_audit->getFilePermission());
if (fd < 0) {
error->assign("Not able to open: " + fileName + ". " \
+ strerror(errno));
return false;
}
fp = fdopen(fd, "w");
fwrite(log.c_str(), log.length(), 1, fp);
fclose(fp);
if (log1.is_open() && log2.is_open()) {
log2 << transaction->toOldAuditLogFormatIndex(fileName, log.length(),
Utils::Md5::hexdigest(log));
log2.flush();
if (m_audit->m_path1.empty() == false
&& m_audit->m_path2.empty() == false) {
std::string msg = transaction->toOldAuditLogFormatIndex(fileName,
log.length(), Utils::Md5::hexdigest(log));
ret = utils::SharedFiles::getInstance().write(m_audit->m_path2, msg,
error);
if (ret == false) {
return false;
}
}
if (log1.is_open() && !log2.is_open()) {
log1 << transaction->toOldAuditLogFormatIndex(fileName, log.length(),
Utils::Md5::hexdigest(log));
log1.flush();
if (m_audit->m_path1.empty() == false
&& m_audit->m_path2.empty() == true) {
std::string msg = transaction->toOldAuditLogFormatIndex(fileName,
log.length(), Utils::Md5::hexdigest(log));
ret = utils::SharedFiles::getInstance().write(m_audit->m_path1, msg,
error);
if (ret == false) {
return false;
}
}
if (!log1.is_open() && log2.is_open()) {
log2 << transaction->toOldAuditLogFormatIndex(fileName, log.length(),
Utils::Md5::hexdigest(log));
log2.flush();
if (m_audit->m_path1.empty() == true
&& m_audit->m_path2.empty() == false) {
std::string msg = transaction->toOldAuditLogFormatIndex(fileName,
log.length(), Utils::Md5::hexdigest(log));
ret = utils::SharedFiles::getInstance().write(m_audit->m_path2, msg,
error);
if (ret == false) {
return false;
}
}
return true;

View File

@@ -18,8 +18,10 @@
#ifndef SRC_AUDIT_LOG_WRITER_PARALLEL_H_
#define SRC_AUDIT_LOG_WRITER_PARALLEL_H_
#include "src/audit_log/writer.h"
#include "src/audit_log/writer/writer.h"
#include "modsecurity/transaction.h"
#include "modsecurity/audit_log.h"
#include "src/utils/shared_files.h"
#ifdef __cplusplus
@@ -28,26 +30,16 @@ namespace audit_log {
namespace writer {
/** @ingroup ModSecurity_CPP_API */
class Parallel : public audit_log::Writer {
class Parallel : public Writer {
public:
explicit Parallel(AuditLog *audit)
: audit_log::Writer(audit) { }
: audit_log::writer::Writer(audit) { }
~Parallel() override;
bool init() override;
bool write(Transaction *transaction, int parts) override;
bool init(std::string *error) override;
bool write(Transaction *transaction, int parts,
std::string *error) override;
void refCountIncrease() override {
m_refereceCount++;
}
void refCountDecreaseAndCheck() override {
m_refereceCount--;
if (m_refereceCount == 0) {
delete this;
}
}
/**
*
@@ -72,8 +64,6 @@ class Parallel : public audit_log::Writer {
YearMonthDayAndTimeFileName = 8,
};
std::ofstream log1;
std::ofstream log2;
inline std::string logFilePath(time_t *t, int part);
};

View File

@@ -15,9 +15,7 @@
#include "src/audit_log/writer/serial.h"
// #include <mutex>
#include "src/audit_log/audit_log.h"
#include "modsecurity/audit_log.h"
namespace modsecurity {
namespace audit_log {
@@ -26,7 +24,7 @@ namespace writer {
Serial::~Serial() {
m_log.close();
utils::SharedFiles::getInstance().close(m_audit->m_path1);
}
@@ -42,25 +40,20 @@ void Serial::generateBoundary(std::string *boundary) {
}
bool Serial::init() {
m_log.open(m_audit->m_path1, std::fstream::out | std::fstream::app);
return true;
bool Serial::init(std::string *error) {
return utils::SharedFiles::getInstance().open(m_audit->m_path1, error);
}
bool Serial::write(Transaction *transaction, int parts) {
bool Serial::write(Transaction *transaction, int parts, std::string *error) {
std::string boundary;
std::string msg;
generateBoundary(&boundary);
msg = transaction->toOldAuditLogFormat(parts, "-" + boundary + "--");
// serialLoggingMutex.lock();
m_log << transaction->toOldAuditLogFormat(parts, "-" + boundary + "--");
m_log.flush();
// serialLoggingMutex.unlock();
return true;
return utils::SharedFiles::getInstance().write(m_audit->m_path1, msg,
error);
}
} // namespace writer

View File

@@ -22,8 +22,10 @@
#ifndef SRC_AUDIT_LOG_WRITER_SERIAL_H_
#define SRC_AUDIT_LOG_WRITER_SERIAL_H_
#include "src/audit_log/writer.h"
#include "src/audit_log/writer/writer.h"
#include "src/utils/shared_files.h"
#include "modsecurity/transaction.h"
#include "modsecurity/audit_log.h"
#ifdef __cplusplus
@@ -33,37 +35,21 @@ namespace writer {
#define SERIAL_AUDIT_LOG_BOUNDARY_LENGTH 8
/** @ingroup ModSecurity_CPP_API */
class Serial : public audit_log::Writer {
class Serial : public Writer {
public:
explicit Serial(audit_log::AuditLog *audit)
: audit_log::Writer(audit) { }
: audit_log::writer::Writer(audit) { }
~Serial() override;
void refCountIncrease() override {
m_refereceCount++;
}
void refCountDecreaseAndCheck() override {
/*
m_refereceCount--;
if (m_refereceCount == 0) {
*/
delete this;
/*
/}
*/
}
bool init() override;;
bool write(Transaction *transaction, int parts) override;
bool init(std::string *error) override;
bool write(Transaction *transaction, int parts,
std::string *error) override;
private:
std::ofstream m_log;
void generateBoundary(std::string *boundary);
};

View File

@@ -13,32 +13,18 @@
*
*/
#include "src/audit_log/writer.h"
#include "src/audit_log/writer/writer.h"
#include <string>
#include "src/audit_log/audit_log.h"
#include "modsecurity/audit_log.h"
namespace modsecurity {
namespace audit_log {
std::string Writer::file_name(const std::string& unique_id) {
time_t timer;
time(&timer);
/** TODO: return file with time stamp and etc. */
return std::string("/tmp/temp_audit_log_file.txt");
}
/**
*
* Temporary print the log into the std::cout to debug purposes.
*
*/
bool Writer::write(Transaction *transaction, int parts) {
std::cout << transaction->toJSON(parts) << std::endl;
return true;
}
namespace writer {
} // namespace writer
} // namespace audit_log
} // namespace modsecurity

View File

@@ -13,47 +13,67 @@
*
*/
#ifndef SRC_AUDIT_LOG_WRITER_H_
#define SRC_AUDIT_LOG_WRITER_H_
#ifndef SRC_AUDIT_LOG_WRITER_WRITER_H_
#define SRC_AUDIT_LOG_WRITER_WRITER_H_
#ifdef __cplusplus
#include <stdio.h>
#include <sys/ipc.h>
#include <sys/shm.h>
#include <sys/types.h>
#include <iostream>
#include <string>
#endif
#include <map>
#include <cstring>
#include "modsecurity/transaction.h"
#include "modsecurity/audit_log.h"
#ifdef __cplusplus
namespace modsecurity {
namespace audit_log {
namespace writer {
class AuditLog;
/** @ingroup ModSecurity_CPP_API */
class Writer {
public:
explicit Writer(AuditLog *audit)
: m_audit(audit),
m_refereceCount(0) { }
m_refereceCount(1) { }
virtual ~Writer() { }
virtual void refCountIncrease() = 0;
virtual void refCountDecreaseAndCheck() = 0;
virtual bool init(std::string *error) = 0;
virtual bool write(Transaction *transaction, int parts,
std::string *error) = 0;
virtual bool init() { return true; }
virtual bool write(Transaction *transaction, int parts);
std::string file_name(const std::string& unique_id);
void refCountIncrease() {
m_refereceCount++;
}
bool refCountDecreaseAndCheck() {
m_refereceCount--;
if (m_refereceCount == 0) {
delete this;
return true;
}
return false;
}
protected:
AuditLog *m_audit;
int m_refereceCount;
};
} // namespace writer
} // namespace audit_log
} // namespace modsecurity
#endif
#endif // SRC_AUDIT_LOG_WRITER_H_
#endif // SRC_AUDIT_LOG_WRITER_WRITER_H_