Compare commits

..

1 Commits

Author SHA1 Message Date
abmohammad
308c598810 test commit 2025-06-23 15:13:12 +03:00
4 changed files with 250 additions and 231 deletions

View File

@@ -50,6 +50,28 @@ static const boost::regex error_log_regex(
", (upstream: \".+?\")"
", (host: \".+?\")$"
);
// Generic regexes for fallback parsing
static const boost::regex generic_crit_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[crit\\] (.+)$"
);
static const boost::regex generic_emerg_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[emerg\\] (.+)$"
);
// Generic regex to extract time, log level and message for fallback parsing
static const boost::regex generic_fallback_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[(\\w+)\\] (.+)$"
);
static const boost::regex server_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+)|(\\w+\\.\\w+)");
static const boost::regex uri_regex("^/");
@@ -57,6 +79,9 @@ static const boost::regex port_regex("\\d+");
static const boost::regex response_code_regex("[0-9]{3}");
static const boost::regex http_method_regex("[A-Za-z]+");
static const string central_nginx_manager = "Central NGINX Manager";
static const string reverse_proxe = "Reverse Proxy";
class NginxMessageReader::Impl
{
public:
@@ -64,6 +89,16 @@ public:
init()
{
dbgFlow(D_NGINX_MESSAGE_READER);
if (Singleton::exists<I_Environment>()) {
auto name = Singleton::Consume<I_Environment>::by<Report>()->get<string>("Service Name");
if (name.ok())
{
dbgInfo(D_NGINX_MESSAGE_READER) << "Service name: " << *name;
service_name = *name;
}
}
I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<NginxMessageReader>();
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
@@ -117,6 +152,12 @@ private:
COUNT
};
struct GenericLogInfo {
string timestamp;
string severity;
string message;
};
void
initSyslogServerSocket()
{
@@ -175,10 +216,10 @@ private:
bool log_sent;
if (isAccessLog(log)) {
log_sent = sendAccessLog(log);
} else if (isAlertErrorLog(log) || isErrorLog(log)) {
} else if (isAlertErrorLog(log) || isErrorLog(log) || isCritErrorLog(log) || isEmergErrorLog(log)) {
log_sent = sendErrorLog(log);
} else {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format for message: "<< log;
continue;
}
if (!log_sent) {
@@ -222,13 +263,22 @@ private:
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Error log" << log;
Maybe<EnumArray<LogInfo, string>> log_info = parseErrorLog(log);
if (!log_info.ok()) {
dbgWarning(D_NGINX_MESSAGE_READER)
<< "Failed parsing the NGINX logs. Error: "
<< log_info.getErr();
return false;
if (log_info.ok()) {
return sendLog(log_info.unpack());
}
return sendLog(log_info.unpack());
if (service_name == central_nginx_manager) {
dbgDebug(D_NGINX_MESSAGE_READER) << "Detailed parsing failed, trying generic parsing";
Maybe<GenericLogInfo> generic_log = parseGenericErrorLog(log);
if (generic_log.ok()) {
return sendGenericLog(generic_log.unpack());
}
}
dbgWarning(D_NGINX_MESSAGE_READER)
<< "Failed parsing the NGINX logs. Error: "
<< log_info.getErr();
return false;
}
bool
@@ -253,7 +303,45 @@ private:
}
bool
sendLog(const EnumArray<LogInfo, string> &log_info)
isCritErrorLog(const string &log) const
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'crit log'. Log: " << log;
return log.find("[crit]") != string::npos;
}
bool
isEmergErrorLog(const string &log) const
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'emerg log'. Log: " << log;
return log.find("[emerg]") != string::npos;
}
string
getCNMEventName(const EnumArray<LogInfo, string> &log_info) const
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name;
switch (log_info[LogInfo::RESPONSE_CODE][0]) {
case '4': {
event_name = "NGINX Proxy Error: Invalid request or incorrect NGINX configuration - Request dropped."
" Please check the reverse proxy configuration of your relevant assets";
break;
}
case '5': {
event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration."
"If the issue persists please contact open-appsec support";
break;
}
default: {
dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code";
return "";
}
}
return event_name;
}
string
getRPMEventName(const EnumArray<LogInfo, string> &log_info) const
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name;
@@ -271,9 +359,45 @@ private:
}
default: {
dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code";
return false;
return "";
}
}
return event_name;
}
string
getServiceName()
{
string service_name = "Unnamed Nano Service";
if (Singleton::exists<I_Environment>()) {
auto name = Singleton::Consume<I_Environment>::by<Report>()->get<string>("Service Name");
if (name.ok()) return *name;
}
return service_name;
}
string getEventName(const EnumArray<LogInfo, string> &log_info)
{
if (service_name == central_nginx_manager)
{
return getCNMEventName(log_info);
}
if (service_name != reverse_proxe)
{
dbgWarning(D_NGINX_MESSAGE_READER)
<< "Unknown service name: "
<< service_name
<< " Response will be sent as RPM";
}
return getRPMEventName(log_info);
}
bool
sendLog(const EnumArray<LogInfo, string> &log_info)
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name = getEventName(log_info);
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Nginx log's event name and response code: "
@@ -283,9 +407,11 @@ private:
LogGen log(
event_name,
ReportIS::Audience::SECURITY,
ReportIS::Severity::INFO,
ReportIS::Severity::HIGH,
ReportIS::Priority::LOW,
ReportIS::Tags::REVERSE_PROXY
service_name == central_nginx_manager ?
ReportIS::Tags::CENTRAL_NGINX_MANAGER :
ReportIS::Tags::REVERSE_PROXY
);
log << LogField("eventConfidence", "High");
@@ -313,6 +439,47 @@ private:
return true;
}
bool
sendGenericLog(const GenericLogInfo &log_info)
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Sending generic log";
// check with christoper
string event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration."
"If the issue persists please contact open-appsec support";
// Convert string severity to ReportIS::Severity
ReportIS::Severity severity = ReportIS::Severity::MEDIUM;
ReportIS::Priority priority = ReportIS::Priority::MEDIUM;
if (log_info.severity == "emerg" || log_info.severity == "crit") {
severity = ReportIS::Severity::CRITICAL;
priority = ReportIS::Priority::URGENT;
} else if (log_info.severity == "error" || log_info.severity == "alert") {
severity = ReportIS::Severity::HIGH;
priority = ReportIS::Priority::HIGH;
}
LogGen log(
event_name,
ReportIS::Audience::SECURITY,
severity,
priority,
ReportIS::Tags::CENTRAL_NGINX_MANAGER
);
log << LogField("eventConfidence", "High");
log << LogField("timestamp", log_info.timestamp);
log << LogField("httpResponseBody", formatGenericLogMessage(log_info));
return true;
}
string
formatGenericLogMessage(const GenericLogInfo &log_info)
{
return "[" + log_info.severity + "] " + log_info.message;
}
bool
sendRateLimitLog(const EnumArray<LogInfo, string> &log_info)
{
@@ -531,6 +698,48 @@ private:
log_info[LogInfo::RULE_NAME] = context.getRuleName();
}
Maybe<GenericLogInfo>
parseGenericErrorLog(const string &log_line)
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Parsing generic error log: " << log_line;
boost::smatch matcher;
GenericLogInfo generic_log;
if (isCritErrorLog(log_line)) {
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_crit_log_regex)) {
const int timestamp_index = 2; // Timestamp from within syslog_regex_string
const int message_index = 5; // The captured message after [crit]
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = "crit";
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
} else if (isEmergErrorLog(log_line)) {
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_emerg_log_regex)) {
const int timestamp_index = 2; // Timestamp from within syslog_regex_string
const int message_index = 5; // The captured message after [emerg]
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = "emerg";
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
}
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_fallback_log_regex)) {
const int timestamp_index = 2; // Timestamp from within syslog_regex_string
const int severity_index = 5; // The captured severity level
const int message_index = 6; // The captured message
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = string(matcher[severity_index].first, matcher[severity_index].second);
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
dbgWarning(D_NGINX_MESSAGE_READER) << "Could not parse log with generic method: " << log_line;
return genError("Could not parse log with generic method");
}
Maybe<EnumArray<LogInfo, string>>
parseErrorLog(const string &log_line)
{
@@ -540,17 +749,29 @@ private:
boost::smatch matcher;
vector<string> result;
boost::regex selected_regex;
// Select appropriate regex based on log type
if (isAlertErrorLog(log_line)) {
selected_regex = alert_log_regex;
} else if (isErrorLog(log_line)) {
selected_regex = error_log_regex;
} else {
dbgWarning(D_NGINX_MESSAGE_READER) << "No matching log type found for log: " << log_line;
return genError("No matching log type found");
}
if (
!NGEN::Regex::regexSearch(
__FILE__,
__LINE__,
log_line,
matcher,
isAlertErrorLog(log_line) ? alert_log_regex : error_log_regex
selected_regex
)
) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
return genError("Unexpected nginx log format");
dbgWarning(D_NGINX_MESSAGE_READER) << "Detailed regex parsing failed for log: " << log_line;
return genError("Detailed regex parsing failed");
}
const int event_message_index = 6;
@@ -591,8 +812,8 @@ private:
addContextFieldsToLogInfo(log_info);
if (!validateLog(log_info)) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
return genError("Unexpected nginx log format");
dbgWarning(D_NGINX_MESSAGE_READER) << "Log validation failed for detailed parsing";
return genError("Log validation failed for detailed parsing");
}
return log_info;
@@ -710,6 +931,7 @@ private:
I_Socket::socketFd syslog_server_socket = -1;
string rate_limit_status_code = "429";
string service_name = "Unnamed Nano Service";
};
NginxMessageReader::NginxMessageReader() : Component("NginxMessageReader"), pimpl(make_unique<Impl>()) {}

View File

@@ -179,10 +179,10 @@ private:
Maybe<void>
configureSyslog()
{
if (!getProfileAgentSettingWithDefault<bool>(false, "centralNginxManagement.syslogEnabled")) {
dbgTrace(D_NGINX_MANAGER) << "Syslog is disabled via settings";
return {};
}
// if (!getProfileAgentSettingWithDefault<bool>(false, "centralNginxManagement.syslogEnabled")) {
// dbgTrace(D_NGINX_MANAGER) << "Syslog is disabled via settings";
// return {};
// }
string syslog_directive = "error_log syslog:server=127.0.0.1:1514 warn;";
auto load_shared_directive_result = loadSharedDirective(syslog_directive);

View File

@@ -1,5 +1,5 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
@@ -13,11 +13,9 @@
#include <iostream>
#include <unistd.h>
#include <sstream>
#include "agent_core_utilities.h"
#include "debug.h"
#include "getopt.h"
#include "internal/shell_cmd.h"
#include "mainloop.h"
#include "nginx_utils.h"
@@ -45,7 +43,6 @@ public:
environment.fini();
time_proxy.fini();
}
private:
ShellCmd shell_cmd;
MainloopComponent mainloop;
@@ -53,153 +50,6 @@ private:
TimeProxyComponent time_proxy;
};
class FogConnection
{
public:
FogConnection(const std::string& token, const std::string& fog)
: var_token(token), var_fog(fog) {}
bool registerAgent() {
std::string curl_cmd = "curl -s --noproxy \"*\" "
"--header \"User-Agent: Infinity Next (a7030abf93a4c13)\" "
"--header \"Content-Type: application/json\" "
"--request POST "
"--data '{\"authenticationData\": [{\"authenticationMethod\": \"token\", \"data\": \""
+ var_token + "\"}], "
"\"metaData\": {\"agentName\": \"ConfCollector\", \"agentType\":"
"\"Embedded\", \"platform\": \"linux\", "
"\"architecture\": \"x86\", \"additionalMetaData\": {\"agentVendor\": \"python\"}}}' "
+ var_fog + "/agents";
std::string response = executeCommand(curl_cmd);
std::string error_check = "echo '" + response + "' | grep referenceId";
std::string error_result = executeCommand(error_check);
if (!error_result.empty()) {
std::cerr << "Couldn't register to the FOG" << std::endl;
return false;
}
agent_id = extractJsonValue(response, "agentId");
clientId = extractJsonValue(response, "clientId");
clientSecret = extractJsonValue(response, "clientSecret");
tenant_id = extractJsonValue(response, "tenantId");
profile_id = extractJsonValue(response, "profileId");
removeNewlines(agent_id);
removeNewlines(clientId);
removeNewlines(clientSecret);
removeNewlines(tenant_id);
removeNewlines(profile_id);
return true;
}
bool getJWT() {
std::string curl_cmd = "curl -s --noproxy \"*\" "
"--header \"User-Agent: Infinity Next (a7030abf93a4c13)\" "
"--header \"Content-Type: application/json\" "
"-d '{\"login\":\"" + clientId + "\", \"password\":\"" + clientSecret + "\"}' "
"--user \"" + clientId + ":" + clientSecret + "\" "
"--request POST "
"--data '{}' "
+ var_fog + "/oauth/token?grant_type=client_credentials";
std::string response = executeCommand(curl_cmd);
std::string error_check = "echo '" + response + "' | grep referenceId";
std::string error_result = executeCommand(error_check);
if (!error_result.empty()) {
std::cerr << "Couldn't receive JWT" << std::endl;
return false;
}
ra_token = extractJsonValue(response, "access_token");
removeNewlines(ra_token);
return true;
}
bool uploadNginxConfig(const std::string& config_file_path)
{
if (tenant_id.empty() || profile_id.empty() || ra_token.empty()) {
std::cerr << "Missing required data for upload (tenant_id, profile_id, or JWT token)" << std::endl;
return false;
}
std::ifstream file_check(config_file_path);
if (!file_check.is_open()) {
std::cerr << "Cannot open config file for upload: " << config_file_path << std::endl;
return false;
}
file_check.close();
std::string upload_url = var_fog + "/agents-core/storage/" + tenant_id + "/" + "nginx/" + profile_id +
"/1/nginx.conf";
std::string curl_cmd = "curl -s --noproxy \"*\" "
"--header \"User-Agent: Infinity Next (a7030abf93a4c13)\" "
"--header \"Authorization: Bearer " + ra_token + "\" "
"--header \"Content-Type: text/plain\" "
"--request PUT "
"--data-binary @" + config_file_path + " "
"-w \"%{http_code}\" "
+ upload_url;
std::string response = executeCommand(curl_cmd);
std::string status_code = "";
if (response.length() >= 3) {
status_code = response.substr(response.length() - 3);
response = response.substr(0, response.length() - 3);
}
if (status_code.empty() || status_code[0] != '2') {
std::cerr << "Upload failed with HTTP status code: " << status_code << std::endl;
return false;
}
std::cout << "Successfully uploaded nginx config to: " << upload_url << " (HTTP " << status_code << ")"
<< std::endl;
return true;
}
private:
std::string var_token;
std::string var_fog;
std::string agent_id;
std::string tenant_id;
std::string profile_id;
std::string ra_token;
std::string clientId;
std::string clientSecret;
std::string executeCommand(const std::string& command) {
std::string result;
FILE* pipe = popen(command.c_str(), "r");
if (!pipe) {
throw std::runtime_error("popen() failed!");
}
char buffer[128];
while (fgets(buffer, sizeof(buffer), pipe) != nullptr) {
result += buffer;
}
pclose(pipe);
return result;
}
std::string extractJsonValue(const std::string& response, const std::string& key) {
std::string command = "echo '" + response + "' | grep -o '\"" + key + "\":\"[^\"]*' | grep -o '[^\"]*$'";
return executeCommand(command);
}
void removeNewlines(std::string& str) {
str.erase(std::remove(str.begin(), str.end(), '\n'), str.end());
str.erase(std::remove(str.begin(), str.end(), '\r'), str.end());
}
};
void
printVersion()
{
@@ -213,16 +63,12 @@ printVersion()
void
printUsage(const char *prog_name)
{
cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" <<
"[--upload --token <token> [--fog <address>]]" << '\n';
cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << '\n';
cout << " -V Print version" << '\n';
cout << " -v Enable verbose output" << '\n';
cout << " -i input_file Specify input file (default is /etc/nginx/nginx.conf)" << '\n';
cout << " -o output_file Specify output file (default is ./full_nginx.conf)" << '\n';
cout << " -h Print this help message" << '\n';
cout << " --upload Upload configuration to FOG (requires --token)" << '\n';
cout << " --token <token> profile token for FOG upload" << '\n';
cout << " --fog <address> FOG server address (default: inext-agents.cloud.ngen.checkpoint.com)" << '\n';
}
int
@@ -230,19 +76,9 @@ main(int argc, char *argv[])
{
string nginx_input_file = "/etc/nginx/nginx.conf";
string nginx_output_file = "full_nginx.conf";
string fog_address = "inext-agents.cloud.ngen.checkpoint.com";
string token;
bool upload_flag = false;
int opt;
static struct option long_options[] = {
{"upload", no_argument, 0, 'u'},
{"token", required_argument, 0, 1001},
{"fog", required_argument, 0, 1002},
{0, 0, 0, 0}
};
while ((opt = getopt_long(argc, argv, "Vvhi:o:", long_options, nullptr)) != -1) {
while ((opt = getopt(argc, argv, "Vvhi:o:h")) != -1) {
switch (opt) {
case 'V':
printVersion();
@@ -259,33 +95,18 @@ main(int argc, char *argv[])
case 'h':
printUsage(argv[0]);
return 0;
case 'u':
upload_flag = true;
break;
case 1001: // --token
token = optarg;
break;
case 1002: // --fog
fog_address = optarg;
break;
default:
printUsage(argv[0]);
return 1;
}
}
for (int i = optind; i < argc; i++) {
for (int i = optind; i < argc;) {
cerr << "Unknown argument: " << argv[i] << '\n';
printUsage(argv[0]);
return 1;
}
if (upload_flag && token.empty()) {
cerr << "Error: --upload requires --token to be specified" << '\n';
printUsage(argv[0]);
return 1;
}
dbgTrace(D_NGINX_MANAGER) << "Starting nginx configuration collector";
MainComponent main_component;
@@ -323,30 +144,5 @@ main(int argc, char *argv[])
cout << "Full nginx configuration file was successfully generated: " << result.unpack() << '\n';
if (upload_flag) {
cout << "Uploading configuration to FOG server: " << fog_address << '\n';
string full_fog_url = fog_address;
if (fog_address.find("http://") != 0 && fog_address.find("https://") != 0) {
full_fog_url = "https://" + fog_address;
}
FogConnection fog_connection(token, full_fog_url);
if (!fog_connection.registerAgent()) {
cerr << "Failed to register agent with the FOG." << '\n';
return 1;
}
if (!fog_connection.getJWT()) {
cerr << "Failed to get JWT token." << '\n';
return 1;
}
if (!fog_connection.uploadNginxConfig(result.unpack())) {
cerr << "Failed to upload nginx config file to FOG." << '\n';
return 1;
}
}
return 0;
}
}

View File

@@ -71,6 +71,7 @@ enum class Tags {
DEPLOYMENT_DOCKER,
WEB_SERVER_SWAG,
WEB_SERVER_NGINX_UNIFIED,
CENTRAL_NGINX_MANAGER,
COUNT
};