diff --git a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc index b996342..6c4407e 100755 --- a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc +++ b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc @@ -285,17 +285,21 @@ Maybe UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const { vector header_values = split(str); - if (header_values.empty()) return genError("No IP found in the xff header list"); vector xff_values = getHeaderValuesFromConfig("x-forwarded-for"); vector cidr_values(xff_values.begin(), xff_values.end()); + string last_valid_ip; for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) { if (!IPAddr::createIPAddr(*it).ok()) { dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it; - return genError("Invalid IP address"); + if (last_valid_ip.empty()) { + return genError("Invalid IP address"); + } + return last_valid_ip; } + last_valid_ip = *it; if (type == ExtractType::PROXYIP) continue; if (!isIpTrusted(*it, cidr_values)) { dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it; @@ -307,7 +311,10 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType typ dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << header_values[0]; - return genError("Invalid IP address"); + if (last_valid_ip.empty()) { + return genError("No Valid Ip address was found"); + } + return last_valid_ip; } return header_values[0]; diff --git a/components/security_apps/central_nginx_manager/central_nginx_manager.cc b/components/security_apps/central_nginx_manager/central_nginx_manager.cc index 6848e95..4c9e6bd 100755 --- a/components/security_apps/central_nginx_manager/central_nginx_manager.cc +++ b/components/security_apps/central_nginx_manager/central_nginx_manager.cc @@ -179,7 +179,7 @@ private: Maybe configureSyslog() { - if (!getProfileAgentSettingWithDefault(true, "centralNginxManagement.syslogEnabled")) { + if (!getProfileAgentSettingWithDefault(false, "centralNginxManagement.syslogEnabled")) { dbgTrace(D_NGINX_MANAGER) << "Syslog is disabled via settings"; return {}; } @@ -331,6 +331,8 @@ public: logError("Could not reload central NGINX configuration. Error: " + reload_result.getErr()); return; } + + logInfo("Central NGINX configuration has been successfully reloaded"); } void @@ -351,11 +353,37 @@ private: { LogGen log( error, + ReportIS::Level::ACTION, ReportIS::Audience::SECURITY, ReportIS::Severity::CRITICAL, - ReportIS::Priority::HIGH, + ReportIS::Priority::URGENT, ReportIS::Tags::POLICY_INSTALLATION ); + + log.addToOrigin(LogField("eventTopic", "Central NGINX Management")); + log << LogField("notificationId", "4165c3b1-e9bc-44c3-888b-863e204c1bfb"); + log << LogField( + "eventRemediation", + "Please verify your NGINX configuration and enforce policy again. " + "Contact Check Point support if the issue persists." + ); + } + + void + logInfo(const string &info) + { + LogGen log( + info, + ReportIS::Level::ACTION, + ReportIS::Audience::SECURITY, + ReportIS::Severity::INFO, + ReportIS::Priority::LOW, + ReportIS::Tags::POLICY_INSTALLATION + ); + + log.addToOrigin(LogField("eventTopic", "Central NGINX Management")); + log << LogField("notificationId", "4165c3b1-e9bc-44c3-888b-863e204c1bfb"); + log << LogField("eventRemediation", "No action required"); } I_MainLoop *i_mainloop = nullptr; diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index b605505..d78ffc7 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -80,7 +80,9 @@ DetailsResolver::Impl::getHostname() Maybe DetailsResolver::Impl::getPlatform() { -#if defined(gaia) +#if defined(gaia_arm) + return string("gaia_arm"); +#elif defined(gaia) return string("gaia"); #elif defined(arm32_rpi) return string("glibc"); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index fe98ccf..2b4ebdf 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -71,7 +71,18 @@ checkPepIdaIdnStatus(const string &command_output) Maybe getRequiredNanoServices(const string &command_output) { - return command_output; + string idaRequiredServices[2] = {"idaSaml", "idaIdn"}; + string platform_str = "gaia"; +#if defined(gaia_arm) + platform_str = "gaia_arm"; +#endif // gaia_arm + string result = ""; + for(const string &serv : idaRequiredServices) { + string add_service = serv + "_" + platform_str; + result = result + add_service + ";"; + } + command_output.empty(); // overcome unused variable + return result; } Maybe @@ -342,6 +353,28 @@ getSMCBasedMgmtName(const string &command_output) return getAttr(command_output, "Mgmt object Name was not found"); } +Maybe +getSmbObjectUid(const string &command_output) +{ + static const char centrally_managed_comd_output = '0'; + + if (command_output.empty() || command_output[0] != centrally_managed_comd_output) { + return genError("Object UUID was not found"); + } + + Maybe obj_uuid = getAttrFromCpsdwanGetDataJson("uuid"); + if (obj_uuid.ok()) { + return obj_uuid.unpack(); + } + + static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C"; + auto file_stream = std::make_shared(obj_path); + if (!file_stream->is_open()) { + return genError("Failed to open the object file"); + } + return getMgmtObjAttr(file_stream, "uuid "); +} + Maybe getSmbObjectName(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 5bebb8a..d2edbb0 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -42,11 +42,6 @@ SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz #ifdef SHELL_CMD_HANDLER #if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) -SHELL_CMD_HANDLER( - "cpProductIntegrationMgmtObjectUid", - "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].uid'", - getMgmtObjUid -) SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry", "FS_PATH=; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] " "&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''", @@ -150,12 +145,17 @@ SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedB SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade) SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal) SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_identity_next_enabled", checkPepIdaIdnStatus) -SHELL_CMD_HANDLER("requiredNanoServices", "echo 'idaSaml_gaia;idaIdn_gaia;'", getRequiredNanoServices) +SHELL_CMD_HANDLER("requiredNanoServices", "echo ida", getRequiredNanoServices) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtObjectName", "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].name'", getMgmtObjName ) +SHELL_CMD_HANDLER( + "cpProductIntegrationMgmtObjectUid", + "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].uid'", + getMgmtObjUid +) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", "cat $FWDIR/database/myself_objects.C " @@ -227,6 +227,11 @@ SHELL_CMD_HANDLER( "cpprod_util FwIsLocalMgmt", getSmbObjectName ) +SHELL_CMD_HANDLER( + "cpProductIntegrationMgmtObjectUid", + "cpprod_util FwIsLocalMgmt", + getSmbObjectUid +) SHELL_CMD_HANDLER( "Application Control", "cat $FWDIR/conf/active_blades.txt | grep -o 'APCL [01]' | cut -d ' ' -f2", diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index ec4d75f..db4729b 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1470,9 +1470,10 @@ private: string cc_opt; tie(config_opt, cc_opt, nginx_version) = nginx_data.unpack(); agent_data_report - << make_pair("nginxVersion", nginx_version) - << make_pair("configureOpt", config_opt) - << make_pair("extraCompilerOpt", cc_opt); + << make_pair("attachmentVersion", "Legacy") + << make_pair("nginxVersion", nginx_version) + << make_pair("configureOpt", config_opt) + << make_pair("extraCompilerOpt", cc_opt); } else { dbgDebug(D_ORCHESTRATOR) << nginx_data.getErr(); } @@ -1528,7 +1529,7 @@ private: } else { curr_agent_data_report = agent_data_report; curr_agent_data_report.disableReportSending(); - agent_data_report << AgentReportFieldWithLabel("report_timestamp", i_time->getWalltimeStr()); + agent_data_report << AgentReportFieldWithLabel("timestamp", i_time->getWalltimeStr()); } } diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index a64b103..b18a8d8 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -208,6 +208,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy MessageMetadata new_config_req_md("127.0.0.1", service_port); new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + new_config_req_md.setSuspension(false); auto res = messaging->sendSyncMessage( HTTPMethod::POST, "/set-new-configuration", diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index 7bbe64a..7a5dbf9 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -113,6 +113,9 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f << parser_depth << " v_len = " << v_len; + + dbgTrace(D_WAAP_DEEP_PARSER) << m_key; + // Decide whether to push/pop the value in the keystack. bool shouldUpdateKeyStack = (flags & BUFFERED_RECEIVER_F_UNNAMED) == 0; @@ -275,13 +278,23 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f // Detect and decode potential base64 chunks in the value before further processing bool base64ParamFound = false; + size_t base64_offset = 0; Waap::Util::BinaryFileType base64BinaryFileType = Waap::Util::BinaryFileType::FILE_TYPE_NONE; if (m_depth == 1 && flags == BUFFERED_RECEIVER_F_MIDDLE && m_key.depth() == 1 && m_key.first() != "#base64"){ dbgTrace(D_WAAP_DEEP_PARSER) << " === will not check base64 since prev data block was not b64-encoded ==="; } else { dbgTrace(D_WAAP_DEEP_PARSER) << " ===Processing potential base64==="; + if (isUrlPayload && m_depth == 1 && cur_val[0] == '/') { + dbgTrace(D_WAAP_DEEP_PARSER) << "removing leading '/' from URL param value"; + base64_offset = 1; + } std::string decoded_val, decoded_key; - base64_variants base64_status = Waap::Util::b64Test(cur_val, decoded_key, decoded_val, base64BinaryFileType); + base64_variants base64_status = Waap::Util::b64Test( + cur_val, + decoded_key, + decoded_val, + base64BinaryFileType, + base64_offset); dbgTrace(D_WAAP_DEEP_PARSER) << " status = " @@ -289,16 +302,50 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f << " key = " << decoded_key << " value = " - << decoded_val; + << decoded_val + << "m_depth = " + << m_depth; switch (base64_status) { case SINGLE_B64_CHUNK_CONVERT: - cur_val = decoded_val; + if (base64_offset) { + cur_val = "/" + decoded_val; + } else { + cur_val = decoded_val; + } base64ParamFound = true; break; + case CONTINUE_DUAL_SCAN: + if (decoded_val.size() > 0) { + decoded_key = "#base64"; + base64ParamFound = false; + if (base64_offset) { + decoded_val = "/" + decoded_val; + } + dbgTrace(D_WAAP_DEEP_PARSER) << m_key; + rc = onKv( + decoded_key.c_str(), + decoded_key.size(), + decoded_val.data(), + decoded_val.size(), + flags, + parser_depth + ); + dbgTrace(D_WAAP_DEEP_PARSER) << "After call to onKv with suspected value rc = " << rc; + dbgTrace(D_WAAP_DEEP_PARSER) << m_key; + break; + } else { + dbgTrace(D_WAAP) << "base64 decode suspected and empty value. Skipping."; + base64ParamFound = false; + break; + } + break; case KEY_VALUE_B64_PAIR: // going deep with new pair in case value is not empty if (decoded_val.size() > 0) { + if (base64_offset) { + decoded_key = "/" + decoded_key; + } cur_val = decoded_val; base64ParamFound = true; rc = onKv( @@ -309,9 +356,13 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f flags, parser_depth ); - - dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc; + dbgTrace(D_WAAP_DEEP_PARSER) << "After call to onKv with suspected value rc = " << rc; + dbgTrace(D_WAAP_DEEP_PARSER) << m_key; if (rc != CONTINUE_PARSING) { + if (shouldUpdateKeyStack) { + m_key.pop("deep parser key"); + } + m_depth--; return rc; } } @@ -323,7 +374,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f } if (base64ParamFound) { - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): pushing #base64 prefix to the key."; + dbgTrace(D_WAAP_DEEP_PARSER) << "pushing #base64 prefix to the key."; m_key.push("#base64", 7, false); } } @@ -437,7 +488,6 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } - m_depth--; return rc; } @@ -587,7 +637,6 @@ DeepParser::parseBuffer( if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } - m_depth--; return DONE_PARSING; } @@ -909,7 +958,6 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( return rc; } } - return rc; } @@ -1081,7 +1129,7 @@ DeepParser::createInternalParser( << " isBodyPayload = " << isBodyPayload; //Detect sensor_data format in body and just use dedicated filter for it - if (m_depth == 1 + if ((m_depth == 1) && isBodyPayload && Waap::Util::detectKnownSource(cur_val) == Waap::Util::SOURCE_TYPE_SENSOR_DATA) { m_parsersDeque.push_back( diff --git a/components/security_apps/waap/waap_clib/KeyStack.cc b/components/security_apps/waap/waap_clib/KeyStack.cc index 979a638..4662c58 100755 --- a/components/security_apps/waap/waap_clib/KeyStack.cc +++ b/components/security_apps/waap/waap_clib/KeyStack.cc @@ -37,14 +37,24 @@ void KeyStack::push(const char* subkey, size_t subkeySize, bool countDepth) { m_nameDepth++; } - dbgTrace(D_WAAP) << "KeyStack(" << m_name << ")::push(): '" << std::string(subkey, subkeySize) << - "' => full_key='" << std::string(m_key.data(), m_key.size()) << "'"; + dbgTrace(D_WAAP) + << "KeyStack(" + << m_name + << ")::push(): '" + << std::string(subkey, subkeySize) + << "' => full_key='" + << std::string(m_key.data(), m_key.size()) + << "'"; } void KeyStack::pop(const char* log, bool countDepth) { // Keep depth balanced even if m_key[] buffer is full if (m_key.empty() || m_stack.empty()) { - dbgDebug(D_WAAP) << "KeyStack(" << m_name << ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! " << log; + dbgDebug(D_WAAP) + << "KeyStack(" + << m_name + << ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! " + << log; return; } @@ -55,6 +65,22 @@ void KeyStack::pop(const char* log, bool countDepth) { // Remove last subkey. m_key.erase(m_stack.back()); m_stack.pop_back(); - dbgTrace(D_WAAP) << "KeyStack(" << m_name << ")::pop(): full_key='" << - std::string(m_key.data(), (int)m_key.size()) << "': pop_key=" << log << "'"; + dbgTrace(D_WAAP) + << "KeyStack(" + << m_name + << ")::pop(): full_key='" + << std::string(m_key.data(), (int)m_key.size()) + << "': pop_key=" + << log + << "'"; +} + +void KeyStack::print(std::ostream &os) const +{ + os + << "KeyStack(" + << m_name + << ")::show(): full_key='" + << std::string(m_key.data(), (int)m_key.size()) + << "'"; } diff --git a/components/security_apps/waap/waap_clib/KeyStack.h b/components/security_apps/waap/waap_clib/KeyStack.h index 6d4105a..bac022d 100755 --- a/components/security_apps/waap/waap_clib/KeyStack.h +++ b/components/security_apps/waap/waap_clib/KeyStack.h @@ -28,6 +28,7 @@ public: void pop(const char* log, bool countDepth=true); bool empty() const { return m_key.empty(); } void clear() { m_key.clear(); m_stack.clear(); } + void print(std::ostream &os) const; size_t depth() const { return m_nameDepth; } size_t size() const { return str().size(); diff --git a/components/security_apps/waap/waap_clib/ParserBase.cc b/components/security_apps/waap/waap_clib/ParserBase.cc index 82785b4..122c1d2 100755 --- a/components/security_apps/waap/waap_clib/ParserBase.cc +++ b/components/security_apps/waap/waap_clib/ParserBase.cc @@ -111,8 +111,7 @@ int BufferedReceiver::onKvDone() // This must be called even if m_value is empty in order to signal the BUFFERED_RECEIVER_F_LAST flag to the // receiver! dbgTrace(D_WAAP_PARSER) - << " Call onKv on the remainder of the buffer not yet pushed to the receiver " - << "calling onKv()"; + << " Call onKv on the remainder of the buffer not yet pushed to the receiver calling onKv()"; int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags, m_parser_depth); // Reset the object's state to allow reuse for other parsers diff --git a/components/security_apps/waap/waap_clib/ParserPDF.cc b/components/security_apps/waap/waap_clib/ParserPDF.cc index e41e7c7..0c6d845 100644 --- a/components/security_apps/waap/waap_clib/ParserPDF.cc +++ b/components/security_apps/waap/waap_clib/ParserPDF.cc @@ -21,6 +21,7 @@ USE_DEBUG_FLAG(D_WAAP); const std::string ParserPDF::m_parserName = "ParserPDF"; const char* PDF_TAIL = "%%EOF"; +const size_t PDF_TAIL_LEN = 5; ParserPDF::ParserPDF( IParserStreamReceiver &receiver, @@ -44,16 +45,21 @@ ParserPDF::push(const char *buf, size_t len) << "' len=" << len; - const char *c; - if (m_state == s_error) { return 0; } - if (len == 0) - { - dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): end of stream. m_state=" << m_state; - if (m_state == s_end) { + if (len == 0) { + dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): end of stream. m_state=" << m_state; + if (m_state == s_body && m_tailOffset >= PDF_TAIL_LEN) { + if (m_receiver.onKey("PDF", 3) != 0) { + m_state = s_error; + return 0; + } + if (m_receiver.onValue("", 0) != 0) { + m_state = s_error; + return 0; + } m_receiver.onKvDone(); } else { m_state = s_error; @@ -61,38 +67,43 @@ ParserPDF::push(const char *buf, size_t len) return 0; } + size_t start = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0; switch (m_state) { case s_start: m_state = s_body; CP_FALL_THROUGH; case s_body: - { - size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0; - c = strstr(buf + tail_lookup_offset, PDF_TAIL); + for (size_t i = start; i < len; i++) { dbgTrace(D_WAAP_PARSER_PDF) - << "string to search: " << std::string(buf + tail_lookup_offset) - << " c=" << c; - if (c) { - m_state = s_end; - CP_FALL_THROUGH; + << "ParserPDF::push(): m_tailOffset=" + << m_tailOffset + << " buf[i]=" + << buf[i]; + if (m_tailOffset <= PDF_TAIL_LEN - 1) { + if (buf[i] == PDF_TAIL[m_tailOffset]) { + m_tailOffset++; + } else { + m_tailOffset = 0; + } } else { - break; + if (buf[i] == '\r' || buf[i] == '\n' || buf[i] == ' ' || buf[i] == 0) { + m_tailOffset++; + } else { + m_tailOffset = 0; + i--; + } } } - case s_end: - if (m_receiver.onKey("PDF", 3) != 0) { - m_state = s_error; - return 0; - } - if (m_receiver.onValue("", 0) != 0) { - m_state = s_error; - return 0; - } + dbgTrace(D_WAAP_PARSER_PDF) + << "ParserPDF::push()->s_body: m_tailOffset=" + << m_tailOffset; break; case s_error: break; default: - dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): unknown state: " << m_state; + dbgTrace(D_WAAP_PARSER_PDF) + << "ParserPDF::push(): unknown state: " + << m_state; m_state = s_error; return 0; } diff --git a/components/security_apps/waap/waap_clib/ParserPDF.h b/components/security_apps/waap/waap_clib/ParserPDF.h index 684888c..f2a6204 100644 --- a/components/security_apps/waap/waap_clib/ParserPDF.h +++ b/components/security_apps/waap/waap_clib/ParserPDF.h @@ -34,7 +34,6 @@ private: enum state { s_start, s_body, - s_end, s_error }; @@ -42,6 +41,7 @@ private: enum state m_state; static const std::string m_parserName; size_t m_parser_depth; + size_t m_tailOffset = 0; }; #endif // __PARSER_PDF_H__ diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index da4dab0..f770342 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -617,6 +617,17 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess() { + bool isBackupSyncEnabled = getProfileAgentSettingWithDefault( + true, + "appsecLearningSettings.backupLocalSync"); + + if (!isBackupSyncEnabled) { + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Local sync is disabled"; + processData(); + saveData(); + return true; + } + RemoteFilesList rawDataFiles; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Getting files of all agents"; diff --git a/components/security_apps/waap/waap_clib/WaapScores.cc b/components/security_apps/waap/waap_clib/WaapScores.cc index 1c707e9..59386ec 100644 --- a/components/security_apps/waap/waap_clib/WaapScores.cc +++ b/components/security_apps/waap/waap_clib/WaapScores.cc @@ -97,7 +97,9 @@ calcIndividualKeywords( std::sort(keywords.begin(), keywords.end()); for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) { - addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray); + addKeywordScore( + scoreBuilder, poolName, *pKeyword, DEFAULT_KEYWORD_SCORE, DEFAULT_KEYWORD_COEF, scoresArray, coefArray + ); } } @@ -112,8 +114,6 @@ calcCombinations( std::vector& keyword_combinations) { keyword_combinations.clear(); - static const double max_combi_score = 1.0f; - double default_coef = 0.8f; for (size_t i = 0; i < keyword_matches.size(); ++i) { std::vector combinations; @@ -137,8 +137,10 @@ calcCombinations( default_score += scoreBuilder.getSnapshotKeywordScore(*it, 0.0f, poolName); } // set default combination score to be the sum of its keywords, bounded by 1 - default_score = std::min(default_score, max_combi_score); - addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray); + default_score = std::min(default_score, DEFAULT_COMBI_SCORE); + addKeywordScore( + scoreBuilder, poolName, combination, default_score, DEFAULT_COMBI_COEF, scoresArray, coefArray + ); keyword_combinations.push_back(combination); } } @@ -155,7 +157,7 @@ calcArrayScore(std::vector& scoreArray) // *pScore is always positive and there's a +10 offset score = 10.0f - left * 10.0f / divisor; } - dbgTrace(D_WAAP_SCORE_BUILDER) << "calculated score: " << score; + dbgDebug(D_WAAP_SCORE_BUILDER) << "calculated score: " << score; return score; } @@ -171,7 +173,9 @@ calcLogisticRegressionScore(std::vector &coefArray, double intercept, do } // Apply the expit function to the log-odds to obtain the probability, // and multiply by 10 to obtain a 'score' in the range [0, 10] - return 1.0f / (1.0f + exp(-log_odds)) * 10.0f; + double score = 1.0f / (1.0f + exp(-log_odds)) * 10.0f; + dbgDebug(D_WAAP_SCORE_BUILDER) << "calculated score (log_odds): " << score << " (" << log_odds << ")"; + return score; } } diff --git a/components/security_apps/waap/waap_clib/WaapScores.h b/components/security_apps/waap/waap_clib/WaapScores.h index d6258d9..5013c80 100644 --- a/components/security_apps/waap/waap_clib/WaapScores.h +++ b/components/security_apps/waap/waap_clib/WaapScores.h @@ -32,6 +32,11 @@ struct ModelLoggingSettings { bool logToStream; }; +static const double DEFAULT_KEYWORD_COEF = 0.3f; +static const double DEFAULT_KEYWORD_SCORE = 2.0f; +static const double DEFAULT_COMBI_COEF = 0.8f; +static const double DEFAULT_COMBI_SCORE = 1.0f; + std::string getScorePoolNameByLocation(const std::string &location); std::string getOtherScorePoolName(); ModelLoggingSettings getModelLoggingSettings(); diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 461f4a8..2f7c2bb 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -40,6 +40,7 @@ #include "WaapOpenRedirectPolicy.h" #include "WaapErrorDisclosurePolicy.h" #include +#include #include "generic_rulebase/parameters_config.h" #include #include "ParserDelimiter.h" @@ -1390,6 +1391,20 @@ Waf2Transaction::findHtmlTagToInject(const char* data, int data_len, int& pos) size_t tagHistPosCheck = m_tagHistPos; for (size_t i=0; i < tagSize; ++i) { if (tag[i] != ::tolower(m_tagHist[tagHistPosCheck])) { + if (i == tagSize - 1 && m_tagHist[tagHistPosCheck] == ' ') { + // match regex on head element with attributes + string dataStr = Waap::Util::charToString(data + pos, data_len - pos); + dataStr = dataStr.substr(0, dataStr.find('>')+1); + tagMatches = NGEN::Regex::regexMatch( + __FILE__, + __LINE__, + dataStr, + boost::regex("(?:\\s+[a-zA-Z_:][-a-zA-Z0-9_:.]*(?:\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s\"'>]*))?)*\\s*>") + ); + pos += dataStr.length() - 1; + dbgTrace(D_WAAP_BOT_PROTECTION) << "matching head element with attributes: " << dataStr << ". match: " << tagMatches; + break; + } tagMatches = false; break; } @@ -1403,12 +1418,8 @@ Waf2Transaction::findHtmlTagToInject(const char* data, int data_len, int& pos) } } - if(!headFound) - { - return false; - } - - return true; + dbgTrace(D_WAAP_BOT_PROTECTION) << "head element tag found: " << headFound; + return headFound; } void @@ -1577,6 +1588,8 @@ Waf2Transaction::decideFinal( dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant API configuration from the I/S"; sitePolicy = &ngenAPIConfig; m_overrideState = getOverrideState(sitePolicy); + + // User limits shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); } else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) { @@ -2322,10 +2335,11 @@ bool Waf2Transaction::decideResponse() bool Waf2Transaction::reportScanResult(const Waf2ScanResult &res) { - if (get_ignoreScore() || (res.score >= SCORE_THRESHOLD && - (m_scanResult == nullptr || res.score > m_scanResult->score))) + if ((get_ignoreScore() || res.score >= SCORE_THRESHOLD) && + (m_scanResult == nullptr || res.score > m_scanResult->score)) { - // Forget any previous scan result and replace with new + dbgTrace(D_WAAP) << "Setting scan result. New score: " << res.score; + // Forget any previous scan result and replace wit, h new delete m_scanResult; m_scanResult = new Waf2ScanResult(res); return true; diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index e22fdc3..b638e72 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -952,6 +952,145 @@ string filterUTF7(const string& text) { return result; } +// Decides the status of a Base64 decoded string based on various parameters. +// @param decoded The decoded string. +// @param entropy The entropy of the original encoded string. +// @param decoded_entropy The entropy of the decoded string. +// @param spacer_count The number of spacer characters in the decoded string. +// @param nonPrintableCharsCount The count of non-printable characters in the decoded string. +// @param clear_on_error Flag indicating whether to clear the decoded string on error. +// @param terminatorCharsSeen The number of terminator characters seen. +// @param called_with_prefix Flag indicating if the function was called with a prefix. +// @return The status of the Base64 decoding process. +// +// Idea: +// Check if input chunk should be replaced by decoded, suspected to be checked both as encoded and decoded +// or cleaned as binary data. Additional case - define as not base64 encoded. +// - in case decoded size less 5 - return invalid +// - check entropy delta based on that base64 encoded data has higher entropy than decoded, usually delta = 0.25 +// - this check should rize suspect but cannot work vice versa +// check if decoded chunk has more than 10% of non-printable characters - this is supect for binary data encoded +// - if no suspect for binary data and entropy is suspected, check empiric conditions to decide if this binary data +// or invalid decoding +// - if suspect for binary data, first check is we have entropy suspection +// - if entropy is suspected and chunk is short and it have more than 25% of nonprintables, return invalid +// since this is not base64 encoded data +// - if entropy is not suspected and chunk is short and it have more than 50% of nonprintables, return invalid +// since this is not base64 encoded data +// - if entropy is suspected and chunk size is between 64-1024, perform additional empiric test +// This test will define if returm value should be treated as suspected or as binary data(cleared) + +base64_decode_status decideStatusBase64Decoded( + string& decoded, + double entropy, + double decoded_entropy, + size_t spacer_count, + size_t nonPrintableCharsCount, + bool clear_on_error, + double terminatorCharsSeen, + bool called_with_prefix +) +{ + base64_decode_status tmp_status = B64_DECODE_OK; + if (entropy - decoded_entropy + terminatorCharsSeen < BASE64_ENTROPY_THRESHOLD_DELTA) { + dbgTrace(D_WAAP_BASE64) + << "The chunk is under suspect to be base64," + << "use dual processing because entropy delta is too low"; + tmp_status = B64_DECODE_SUSPECTED; + } + + bool empiric_condition = false; + if (decoded.size() >= 5) { + if (spacer_count > 1) { + nonPrintableCharsCount = nonPrintableCharsCount - spacer_count + 1; + } + dbgTrace(D_WAAP_BASE64) + << "(before test for unprintables): decoded.size=" + << decoded.size() + << ", nonPrintableCharsCount=" + << nonPrintableCharsCount + << ", clear_on_error=" + << clear_on_error + << ", called_with_prefix=" + << called_with_prefix; + if (nonPrintableCharsCount * 10 < decoded.size()) { + dbgTrace(D_WAAP_BASE64) + << "(decode/replace due to small amount of nonprintables): will decide based on entropy values"; + } else { // more than 10% of non-printable characters + dbgTrace(D_WAAP_BASE64) << "large amount of nonporintables"; + if (tmp_status == B64_DECODE_SUSPECTED) { + // entropy - decoded_entropy + terminatorCharsSeen < 0.25 + if (decoded.size() < 16 && nonPrintableCharsCount * 4 > decoded.size()) { + decoded.clear(); + return B64_DECODE_INVALID; + } + dbgTrace(D_WAAP_BASE64) + << "(large amount of nonporintables + entropy suspect), check emprirics because decoded." + << " terminatorCharsSeen=" + << terminatorCharsSeen; + // empiric test based on investigation of real payloads + empiric_condition = entropy < decoded_entropy + && entropy > BASE64_ENTROPY_BASE_THRESHOLD + && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD + && !called_with_prefix + && decoded.size() > BASE64_MIN_SIZE_LIMIT + && decoded.size() < BASE64_MAX_SIZE_LIMIT + && terminatorCharsSeen != 0; + if (!empiric_condition) { + if (clear_on_error) decoded.clear(); + return B64_DECODE_SUSPECTED; + } else { + if (clear_on_error) decoded.clear(); + tmp_status = B64_DECODE_OK; + } + } else { // entropy - decoded_entropy + terminatorCharsSeen >= 0.25 + // one more empiric based on uT and real payloads + if (decoded.size() < 16 + && nonPrintableCharsCount * 2 > decoded.size() + && terminatorCharsSeen == 0) { + decoded.clear(); + return B64_DECODE_INVALID; + } + dbgTrace(D_WAAP_BASE64) + << "(delete as binary content) because decoded. Return B64_DECODE_INCOMPLETE"; + if (clear_on_error) decoded.clear(); + return B64_DECODE_INCOMPLETE; + } + } // less than 10% of non-printable characters + dbgTrace(D_WAAP_BASE64) + << "After handling unprintables checking status"; + if (tmp_status == B64_DECODE_OK) { + dbgTrace(D_WAAP_BASE64) << "replacing with decoded data, return B64_DECODE_OK"; + return B64_DECODE_OK; + } else { // tmp_status == B64_DECODE_SUSPECTED, entropy - decoded_entropy + terminatorCharsSeen < 0.25 + dbgTrace(D_WAAP_BASE64) << "Suspected due to entropy, making empiric test"; + // and one more empiric test based on investigation of real payloads + empiric_condition = entropy < decoded_entropy + && entropy > BASE64_ENTROPY_BASE_THRESHOLD + && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD + && !called_with_prefix + && decoded.size() > BASE64_MIN_SIZE_LIMIT + && decoded.size() < BASE64_MAX_SIZE_LIMIT; + if (empiric_condition) { + dbgTrace(D_WAAP_BASE64) << "Empiric test failed, non-base64 chunk, return B64_DECODE_INVALID"; + decoded.clear(); + return B64_DECODE_INVALID; + } + dbgTrace(D_WAAP_BASE64) << "Empiric test passed, return B64_DECODE_SUSPECTED"; + return B64_DECODE_SUSPECTED; + } + return B64_DECODE_OK; // successfully decoded. Returns decoded data in "decoded" parameter + } + + // If decoded size is too small - leave the encoded value (return false) + decoded.clear(); // discard partial data + dbgTrace(D_WAAP_BASE64) + << "(leave as-is) because decoded too small. decoded.size=" + << decoded.size(); + return B64_DECODE_INVALID; +} + + // Attempts to validate and decode base64-encoded chunk. // Value is the full value inside which potential base64-encoded chunk was found, // it and end point to start and end of that chunk. @@ -980,18 +1119,28 @@ base64_decode_status decodeBase64Chunk( uint32_t spacer_count = 0; uint32_t length = end - it; - dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'"; + dbgTrace(D_WAAP) + << "value='" + << value + << "' match='" + << string(it, end) + << "' clear_on_error='" + << clear_on_error + << "' called_with_prefix='" + << called_with_prefix + << "'"; string::const_iterator begin = it; // The encoded data length (without the "base64," prefix) should be exactly divisible by 4 // len % 4 is not 0 i.e. this is not base64 - if ((end - it) % 4 != 0) { - dbgTrace(D_WAAP_BASE64) << - "b64DecodeChunk: (leave as-is) because encoded data length should be exactly divisible by 4."; + if ((end - it) % 4 == 1) { + dbgTrace(D_WAAP_BASE64) + << "(leave as-is) because encoded data length should not be <4*x + 1>."; return B64_DECODE_INVALID; } - std::unordered_map frequency; + std::unordered_map original_occurences_counter; + std::unordered_map decoded_occurences_counter; while (it != end) { unsigned char c = *it; @@ -999,9 +1148,8 @@ base64_decode_status decodeBase64Chunk( if (terminatorCharsSeen) { // terminator characters must all be '=', until end of match. if (c != '=') { - dbgTrace(D_WAAP_BASE64) << - "decodeBase64Chunk: (leave as-is) because terminator characters must all be '='," << - "until end of match."; + dbgTrace(D_WAAP_BASE64) + << "(leave as-is) because terminator characters must all be '=' until end of match."; return B64_DECODE_INVALID; } @@ -1009,13 +1157,13 @@ base64_decode_status decodeBase64Chunk( terminatorCharsSeen++; if (terminatorCharsSeen > 2) { - dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because terminatorCharsSeen > 2"; + dbgTrace(D_WAAP_BASE64) << "(leave as-is) because terminatorCharsSeen > 2"; return B64_DECODE_INVALID; } // allow for more terminator characters it++; - frequency[c]++; + original_occurences_counter[c]++; continue; } @@ -1040,12 +1188,18 @@ base64_decode_status decodeBase64Chunk( // Start tracking terminator characters terminatorCharsSeen++; it++; - frequency[c]++; + original_occurences_counter[c]++; continue; } else { - dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because of non-base64 character ('" << - c << "', ASCII " << (unsigned int)c << ", offset " << (it-begin) << ")"; + dbgTrace(D_WAAP_BASE64) + << "(leave as-is) because of non-base64 character ('" + << c + << "', ASCII " + << (unsigned int)c + << ", offset " + << (it-begin) + << ")"; return B64_DECODE_INVALID; // non-base64 character } @@ -1068,18 +1222,19 @@ base64_decode_status decodeBase64Chunk( } decoded += (char)code; + decoded_occurences_counter[(char)code]++; } it++; - frequency[c]++; + original_occurences_counter[c]++; } // end of encoded sequence decoded. dbgTrace(D_WAAP_BASE64) - << "decodeBase64Chunk: decoded.size=" + << "decoding done: decoded.size=" << decoded.size() - << ", nonPrintableCharsCount=" + << ", uncorrected nonPrintableCharsCount=" << nonPrintableCharsCount << ", spacer_count = " << spacer_count @@ -1088,56 +1243,42 @@ base64_decode_status decodeBase64Chunk( << "; decoded='" << decoded << "'"; - // Check if entropy is correlates with b64 threshold (initially > 4.5) - if (!called_with_prefix) { - double entropy = 0; - double p = 0; - for (const auto& pair : frequency) { - p = pair.second / length; - entropy -= p * std::log2(p); - } - dbgTrace(D_WAAP_BASE64) << " ===b64Test===: base entropy = " << entropy << "length = " << length; - // Add short payload factor - if (length < 16) - entropy = entropy * 16 / length; - // Enforce tailoring '=' characters - entropy+=terminatorCharsSeen; - - dbgTrace(D_WAAP_BASE64) << " ===b64Test===: corrected entropy = " << entropy << "length = " << length; - if (entropy <= base64_entropy_threshold) { - return B64_DECODE_INVALID; - } + double entropy = 0; + double p = 0; + double decoded_entropy = 0; + for (const auto& pair : original_occurences_counter) { + p = pair.second / length; + entropy -= p * std::log2(p); } - - // Return success only if decoded.size>=5 and there are less than 10% of non-printable - // characters in output. - if (decoded.size() >= 5) { - if (spacer_count > 1) { - nonPrintableCharsCount = nonPrintableCharsCount - spacer_count + 1; - } - if (nonPrintableCharsCount * 10 < decoded.size()) { - dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (decode/replace) decoded.size=" << decoded.size() << - ", nonPrintableCharsCount=" << nonPrintableCharsCount << ": replacing with decoded data"; - } - else { - dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (delete) because decoded.size=" << decoded.size() << - ", nonPrintableCharsCount=" << nonPrintableCharsCount << - ", clear_on_error=" << clear_on_error; - if (clear_on_error) decoded.clear(); - return B64_DECODE_INCOMPLETE; - } - dbgTrace(D_WAAP_BASE64) << "returning true: successfully decoded." - << " Returns decoded data in \"decoded\" parameter"; - return B64_DECODE_OK; // successfully decoded. Returns decoded data in "decoded" parameter + for (const auto &pair : decoded_occurences_counter) { + p = pair.second / decoded.size(); + decoded_entropy -= p * std::log2(p); } + dbgTrace(D_WAAP_BASE64) + << "Base entropy = " + << entropy + << " Decoded_entropy = " + << decoded_entropy + << "length = " + << length; + + base64_decode_status return_status = decideStatusBase64Decoded( + decoded, + entropy, + decoded_entropy, + spacer_count, + nonPrintableCharsCount, + clear_on_error, + terminatorCharsSeen, + called_with_prefix + ); + + dbgTrace(D_WAAP_BASE64) + << "After decideStatusBase64Decoded return_status=" + << return_status; + + return return_status; - // If decoded size is too small - leave the encoded value (return false) - decoded.clear(); // discard partial data - dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because decoded too small. decoded.size=" << - decoded.size() << - ", nonPrintableCharsCount=" << nonPrintableCharsCount << - ", clear_on_error=" << clear_on_error; - return B64_DECODE_INVALID; } // Attempts to detect and validate base64 chunk. @@ -1180,8 +1321,9 @@ b64DecodeChunk( return false; } } - - return decodeBase64Chunk(value, it, end, decoded) != B64_DECODE_INVALID; + base64_decode_status status = decodeBase64Chunk(value, it, end, decoded); + dbgTrace(D_WAAP_BASE64) << "b64DecodeChunk: status = " << status; + return status != B64_DECODE_INVALID; } vector split(const string& s, char delim) { @@ -1281,6 +1423,7 @@ static void b64TestChunk(const string &s, int &deletedCount, string &outStr) { + dbgTrace(D_WAAP_BASE64) << " ===b64TestChunk===: starting with = '" << s << "'"; size_t chunkLen = (chunkEnd - chunkStart); if ((chunkEnd - chunkStart) > static_cast(b64_prefix.size()) && @@ -1289,11 +1432,9 @@ static void b64TestChunk(const string &s, chunkLen -= b64_prefix.size(); } - size_t chunkRem = chunkLen % 4; - - // Only match chunk whose length is divisible by 4 string repl; - if (chunkRem == 0 && cb(s, chunkStart, chunkEnd, repl)) { + dbgTrace(D_WAAP_BASE64) << " ===b64TestChunk===: chunkLen = " << chunkLen; + if (cb(s, chunkStart, chunkEnd, repl)) { // Succesfully matched b64 chunk if (!repl.empty()) { outStr += repl; @@ -1340,9 +1481,7 @@ bool detectBase64Chunk( dbgTrace(D_WAAP_BASE64) << " ===detectBase64Chunk===: isB64AlphaChar = true, '" << *it << "'"; start = it; end = s.end(); - if ((end - start) % 4 == 0) { - return true; - } + return true; } // non base64 before supposed chunk - will not process return false; @@ -1381,17 +1520,31 @@ bool isBase64PrefixProcessingOK ( if (detectBase64Chunk(s, start, end)) { dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected"; if ((start != s.end()) && (end == s.end())) { + dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected but not complete"; retVal = processDecodedChunk(s, start, end, value, binaryFileType, true); + dbgTrace(D_WAAP_BASE64) + << " ===isBase64PrefixProcessingOK===: after processDecodedChunk retVal = " + << retVal + << " binaryFileType = " + << binaryFileType; } } else if (start != s.end()) { - dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected." - " searching for known file header only"; + dbgTrace(D_WAAP_BASE64) + << " ===isBase64PrefixProcessingOK===: chunk not detected. searching for known file header only"; end = (start + MAX_HEADER_LOOKUP < s.end()) ? start + MAX_HEADER_LOOKUP : s.end(); processDecodedChunk(s, start, end, value, binaryFileType); value.clear(); + dbgTrace(D_WAAP_BASE64) + << " ===isBase64PrefixProcessingOK===: after processDecodedChunk binaryFileType = " + << binaryFileType; return binaryFileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE; } } + dbgTrace(D_WAAP_BASE64) + << " ===isBase64PrefixProcessingOK===: retVal = " + << retVal + << " binaryFileType = " + << binaryFileType; return retVal != B64_DECODE_INVALID; } @@ -1399,23 +1552,31 @@ base64_variants b64Test ( const string &s, string &key, string &value, - BinaryFileType &binaryFileType) + BinaryFileType &binaryFileType, + const size_t offset) { key.clear(); - bool retVal; binaryFileType = Waap::Util::BinaryFileType::FILE_TYPE_NONE; + auto begin = s.begin() + offset; + dbgTrace(D_WAAP_BASE64) + << " ===b64Test===: string = " + << s + << " key = " + << key + << " value = " + << value + << " offset = " + << offset; - dbgTrace(D_WAAP_BASE64) << " ===b64Test===: string = " << s - << " key = " << key << " value = " << value; // Minimal length - if (s.size() < 8) { + if (s.size() < 8 + offset) { return CONTINUE_AS_IS; } dbgTrace(D_WAAP_BASE64) << " ===b64Test===: minimal lenght test passed"; std::string prefix_decoded_val; - string::const_iterator it = s.begin(); + auto it = begin; // 1st check if we have key candidate if (base64_key_value_detector_re.hasMatch(s)) { @@ -1433,7 +1594,7 @@ base64_variants b64Test ( break; case EQUAL: if (*it == '=') { - it = s.begin(); + it = begin; state=MISDETECT; continue; } @@ -1455,7 +1616,7 @@ base64_variants b64Test ( if (it == s.end() || state == MISDETECT) { dbgTrace(D_WAAP_BASE64) << " ===b64Test===: detected *it = s.end()" << *it; if (key.size() > 0) { - it = s.begin(); + it = begin; key.clear(); } } else { @@ -1479,7 +1640,7 @@ base64_variants b64Test ( } } - string::const_iterator start = s.end(); + auto start = s.end(); dbgTrace(D_WAAP_BASE64) << " ===b64Test===: B64 itself = " << *it << " ======="; bool isB64AlphaChar = Waap::Util::isAlphaAsciiFast(*it) || isdigit(*it) || *it=='/' || *it=='+'; if (isB64AlphaChar) { @@ -1487,11 +1648,6 @@ base64_variants b64Test ( dbgTrace(D_WAAP_BASE64) << " ===b64Test===: Start tracking potential b64 chunk = " << *it << " ======="; start = it; - if ((s.end() - start) % 4 != 0) { - key.clear(); - value.clear(); - return CONTINUE_AS_IS; - } } else { dbgTrace(D_WAAP_BASE64) << @@ -1512,17 +1668,37 @@ base64_variants b64Test ( key.pop_back(); dbgTrace(D_WAAP_BASE64) << " ===b64Test===: FINAL key = '" << key << "'"; } - retVal = decodeBase64Chunk(s, start, s.end(), value) != B64_DECODE_INVALID; + base64_decode_status decode_chunk_status = decodeBase64Chunk(s, start, s.end(), value); - dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After testing and conversion value = " - << value << "retVal = '" << retVal <<"'"; - if (!retVal) { + dbgTrace(D_WAAP_BASE64) + << " ===b64Test===: After testing and conversion value = " + << value + << "decode_chunk_status = '" + << decode_chunk_status + <<"'"; + if (decode_chunk_status == B64_DECODE_INVALID) { key.clear(); value.clear(); return CONTINUE_AS_IS; } - dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After tpassed retVal check = " - << value << "retVal = '" << retVal <<"'" << "key = '" << key << "'"; + + if (decode_chunk_status == B64_DECODE_INCOMPLETE) { + value.clear(); + } + + if (decode_chunk_status == B64_DECODE_SUSPECTED) { + return CONTINUE_DUAL_SCAN; + } + + dbgTrace(D_WAAP_BASE64) + << " ===b64Test===: After tpassed retVal check = " + << value + << "decode_chunk_status = '" + << decode_chunk_status + <<"'" + << "key = '" + << key + << "'"; if (key.empty()) { return SINGLE_B64_CHUNK_CONVERT; } else { @@ -1548,7 +1724,7 @@ void b64Decode( deletedCount = 0; outStr = ""; int offsetFix = 0; - + dbgTrace(D_WAAP_BASE64) << " ===b64Decode===: starting with = '" << s << "'"; string::const_iterator it = s.begin(); // Minimal length @@ -1596,6 +1772,11 @@ void b64Decode( } // Decode and add chunk + dbgTrace(D_WAAP_BASE64) + << " ===b64Decode===: chunkStart = " + << *chunkStart + << " it = " + << *it; b64TestChunk(s, chunkStart, it, cb, decodedCount, deletedCount, outStr); // stop tracking b64 chunk @@ -1607,6 +1788,7 @@ void b64Decode( } if (chunkStart != s.end()) { + dbgTrace(D_WAAP_BASE64) << " ===b64Decode===: chunkStart = " << *chunkStart; b64TestChunk(s, chunkStart, it, cb, decodedCount, deletedCount, outStr); } } diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index d2d8cdf..5c42b59 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -32,9 +32,15 @@ #define GCC_VERSION (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) -enum base64_variants {SINGLE_B64_CHUNK_CONVERT, KEY_VALUE_B64_PAIR, CONTINUE_AS_IS}; +enum base64_variants {SINGLE_B64_CHUNK_CONVERT, KEY_VALUE_B64_PAIR, CONTINUE_AS_IS, CONTINUE_DUAL_SCAN}; enum base64_stage {BEFORE_EQUAL, EQUAL, DONE, MISDETECT}; -enum base64_decode_status {B64_DECODE_INVALID, B64_DECODE_OK, B64_DECODE_INCOMPLETE}; +enum base64_decode_status {B64_DECODE_INVALID, B64_DECODE_OK, B64_DECODE_INCOMPLETE, B64_DECODE_SUSPECTED}; + +#define BASE64_ENTROPY_BASE_THRESHOLD 5.0 +#define BASE64_ENTROPY_DECODED_THRESHOLD 5.4 +#define BASE64_ENTROPY_THRESHOLD_DELTA 0.25 +#define BASE64_MIN_SIZE_LIMIT 16 +#define BASE64_MAX_SIZE_LIMIT 1024 // This is portable version of stricmp(), which is non-standard function (not even in C). // Contrary to stricmp(), for a slight optimization, s2 is ASSUMED to be already in lowercase. @@ -865,6 +871,17 @@ void unescapeUnicode(std::string &text); // Try to find and decode UTF7 chunks std::string filterUTF7(const std::string &text); +base64_decode_status +decideStatusBase64Decoded( + std::string& decoded, + double entropy, + double decoded_entropy, + size_t spacer_count, + size_t nonPrintableCharsCount, + bool clear_on_error, + double terminatorCharsSeen, + bool called_with_prefix); + base64_decode_status decodeBase64Chunk( const std::string &value, @@ -926,7 +943,8 @@ namespace Util { const std::string &s, std::string &key, std::string &value, - BinaryFileType &binaryFileType); + BinaryFileType &binaryFileType, + size_t offset = 0); // The original stdlib implementation of isalpha() supports locale settings which we do not really need. // It is also proven to contribute to slow performance in some of the algorithms using it. diff --git a/components/utils/nginx_utils/nginx_utils.cc b/components/utils/nginx_utils/nginx_utils.cc index e0af8fa..beefd3c 100755 --- a/components/utils/nginx_utils/nginx_utils.cc +++ b/components/utils/nginx_utils/nginx_utils.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include "debug.h" #include "maybe_res.h" @@ -75,13 +76,13 @@ NginxConfCollector::expandIncludes(const string &include_pattern) const { struct dirent *entry; while ((entry = readdir(dir)) != nullptr) { if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue; - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, entry->d_name, pattern)) { matching_files.push_back(maybe_directory + "/" + entry->d_name); dbgTrace(D_NGINX_MANAGER) << "Matched file: " << maybe_directory << '/' << entry->d_name; } } closedir(dir); + sort(matching_files.begin(), matching_files.end()); return matching_files; } diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index f88a33b..49b51f8 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -436,6 +436,7 @@ AgentDetails::loadProxyType(const string &proxy_type) } #ifdef gaia + (void)proxy_type; I_ShellCmd *shell_cmd = Singleton::Consume::by(); auto proxy_ip = shell_cmd->getExecOutput("dbget proxy:ip-address| tr -d '\n'"); if (!proxy_ip.ok()) return proxy_ip; diff --git a/core/config/config.cc b/core/config/config.cc index b77f6d4..31db962 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -203,6 +203,7 @@ private: MessageMetadata service_config_req_md("127.0.0.1", 7777); service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + service_config_req_md.setSuspension(false); auto service_config_status = messaging->sendSyncMessage( HTTPMethod::POST, "/set-nano-service-config", @@ -214,6 +215,7 @@ private: MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + secondary_port_req_md.setSuspension(false); service_config_status = messaging->sendSyncMessage( HTTPMethod::POST, "/set-nano-service-config", @@ -251,6 +253,7 @@ private: MessageMetadata service_config_req_md("127.0.0.1", 7777); service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + service_config_req_md.setSuspension(false); bool service_config_status = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", @@ -262,6 +265,7 @@ private: MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + secondary_port_req_md.setSuspension(false); service_config_status = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index 3588853..d7dd483 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -527,7 +527,7 @@ Debug::preload() active_streams["FOG"] = make_shared(); string branch = Version::getBranch(); - if (branch == "master" || branch.substr(0, 6) == "hotfix") { + if (branch == "open-source" || branch == "master" || branch.substr(0, 6) == "hotfix") { should_assert_optional = false; } else { should_assert_optional = true; diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h index a016d8f..af7ef27 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -69,14 +69,16 @@ public: uint16_t _port_num, Flags _conn_flags, bool _should_buffer = false, - bool _is_to_fog = false + bool _is_to_fog = false, + bool _should_suspend = true ) : host_name(_host_name), port_num(_port_num), conn_flags(_conn_flags), should_buffer(_should_buffer), is_to_fog(_is_to_fog), - should_send_access_token(true) + should_send_access_token(true), + should_suspend(_should_suspend) {} const bool & @@ -193,6 +195,12 @@ public: is_dual_auth = true; } + void + setSuspension(bool _should_suspend) + { + should_suspend = _should_suspend; + } + void setExternalCertificate(const std::string &_external_certificate) { @@ -211,6 +219,12 @@ public: return should_buffer; } + bool + shouldSuspend() const + { + return should_suspend; + } + bool isProxySet() const { @@ -314,6 +328,7 @@ private: bool is_rate_limit_block = false; uint rate_limit_block_time = 0; bool should_send_access_token = true; + bool should_suspend = true; }; #endif // __MESSAGING_METADATA_H__ diff --git a/core/intelligence_is_v2/intelligence_request.cc b/core/intelligence_is_v2/intelligence_request.cc index f9dac10..bec3054 100644 --- a/core/intelligence_is_v2/intelligence_request.cc +++ b/core/intelligence_is_v2/intelligence_request.cc @@ -22,7 +22,7 @@ using namespace std; USE_DEBUG_FLAG(D_INTELLIGENCE); -static const unsigned int upper_assets_limit = 50; +static const unsigned int upper_assets_limit = 200; static const unsigned int upper_confidence_limit = 1000; Maybe diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index bb5a8fc..1e72482 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -125,7 +125,9 @@ MessagingComp::sendMessage( } Connection conn = maybe_conn.unpack(); - if (conn.isSuspended()) return suspendMessage(body, method, uri, category, message_metadata); + if (message_metadata.shouldSuspend() && conn.isSuspended()) { + return suspendMessage(body, method, uri, category, message_metadata); + } bool is_to_fog = isMessageToFog(message_metadata); auto metadata = message_metadata; diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index ca1eb85..e276fb4 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -110,8 +110,14 @@ if [ $retval -eq 0 ]; then fi if [ $var_gaia_release -eq 0 ] || [ $var_mds_release -eq 0 ]; then - var_arch="gaia" - var_arch_flag="--gaia" + arch=$(uname -a | awk '{print $(NF -1) }') + if test "${arch}" == "aarch64"; then + var_arch="gaia_arm" + var_arch_flag="--gaia_arm" + else + var_arch="gaia" + var_arch_flag="--gaia" + fi elif [ $var_alpine_release -eq 0 ]; then var_is_alpine=true else @@ -322,7 +328,7 @@ while true; do LOG_FILE_PATH=$1 fi echo "Log files path: ${LOG_FILE_PATH}" - elif [ "$1" = "--arm64_trustbox" ] || [ "$1" = "--arm64_linaro" ] || [ "$1" = "--arm32_rpi" ] || [ "$1" = "--gaia" ] || [ "$1" = "--smb_mrv_v1" ] || [ "$1" = "--smb_sve_v2" ] || [ "$1" = "--smb_thx_v3" ] || [ "$1" = "--x86" ] || [ "$1" = "./orchestration_package.sh" ]; then + elif [ "$1" = "--arm64_trustbox" ] || [ "$1" = "--arm64_linaro" ] || [ "$1" = "--arm32_rpi" ] || [ "$1" = "--gaia" ] || [ "$1" = "--gaia_arm" ] || [ "$1" = "--smb_mrv_v1" ] || [ "$1" = "--smb_sve_v2" ] || [ "$1" = "--smb_thx_v3" ] || [ "$1" = "--x86" ] || [ "$1" = "./orchestration_package.sh" ]; then shift continue elif [ "$1" = "--skip_registration" ]; then @@ -416,7 +422,7 @@ if command -v which &>/dev/null; then var_which_cmd_exists=1 fi -if [ $var_arch != "gaia" ] && [ $var_which_cmd_exists -eq 1 ]; then +if [ $var_arch != "gaia" ] && [ $var_arch != "gaia_arm" ] && [ $var_which_cmd_exists -eq 1 ]; then if [ -n "$(which systemctl)" ]; then var_startup_service="systemd" else @@ -490,12 +496,12 @@ cp_copy() # Initials - cc update_cloudguard_appsec_manifest() { - if [ -z ${CLOUDGUARD_APPSEC_STANDALONE} ] || [ -z ${DOCKER_RPM_ENABLED} ]; then + if [ -z ${INFINITY_NEXT_NANO_AGENT} ] && { [ -z ${CLOUDGUARD_APPSEC_STANDALONE} ] || [ -z ${DOCKER_RPM_ENABLED} ]; }; then return fi selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/cloudguard_appsec_manifest.json" - if [ "${DOCKER_RPM_ENABLED}" = "false" ]; then + if [ "${DOCKER_RPM_ENABLED}" = "false" ] || [ "${INFINITY_NEXT_NANO_AGENT}" = "TRUE" ]; then selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/self_managed_cloudguard_appsec_manifest.json" fi @@ -564,7 +570,7 @@ install_watchdog_gaia() # Add cp-nano-watchdog to DB dbset process:${watchdog_pm_name} t dbset process:${watchdog_pm_name}:path ${FILESYSTEM_PATH}/${WATCHDOG_PATH} - dbset process:${watchdog_pm_name}:arg:1 --gaia + dbset process:${watchdog_pm_name}:arg:1 ${var_arch_flag} dbset process:${watchdog_pm_name}:runlevel 1 dbset :save tellpm ${watchdog_pm_name} t @@ -616,7 +622,7 @@ install_watchdog() cp_copy service/smb/nano_agent.init /storage/nano_agent/etc/nano_agent.init chmod +rx /storage/nano_agent/etc/nano_agent.init elif [ $var_container_mode = false ]; then - if [ $var_arch = "gaia" ]; then + if [ $var_arch = "gaia" ] || [ $var_arch = "gaia_arm" ]; then cp_exec "ln -s ${FWDIR}/bin/curl_cli ${FWDIR}/bin/curl" cp_exec "ln -s ${CPDIR}/bin/cpopenssl ${CPDIR}/bin/openssl" cp_copy watchdog/access_pre_init $INIT_D_PATH/access_pre_init @@ -657,7 +663,7 @@ install_watchdog() cp_exec "$INIT_D_PATH/nano_agent.init start" elif [ "$is_smb" = "1" ]; then cp_exec "/storage/nano_agent/etc/nano_agent.init start" - elif [ $var_arch = "gaia" ]; then + elif [ $var_arch = "gaia" ] || [ $var_arch = "gaia_arm" ]; then install_watchdog_gaia else cp_exec "service $NANO_AGENT_SERVICE_NAME start" @@ -785,7 +791,7 @@ upgrade_conf_if_needed() cp_exec "cp -f configuration/orchestration.cfg ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" execution_flags="execution_flags=\"--orchestration-mode=${var_orchestration_mode}\"" echo $execution_flags >> ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg - if [ $var_arch = "gaia" -o "$is_smb" = "1" ]; then + if [ $var_arch = "gaia" -o $var_arch = "gaia_arm" -o "$is_smb" = "1" ]; then if [ -z "${gaia_ld_path}" ]; then gaia_ld_path="${LD_LIBRARY_PATH}" fi @@ -1001,7 +1007,7 @@ install_orchestration() cp_exec "cp -f configuration/orchestration.cfg ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" execution_flags="execution_flags=\"--orchestration-mode=${var_orchestration_mode}\"" echo $execution_flags >> ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg - if [ $var_arch = "gaia" -o "$is_smb" = "1" ]; then + if [ $var_arch = "gaia" -o $var_arch = "gaia_arm" -o "$is_smb" = "1" ]; then if [ -z "${gaia_ld_path}" ]; then gaia_ld_path="${LD_LIBRARY_PATH}" fi @@ -1101,7 +1107,14 @@ install_orchestration() if [ -z "${var_token}" ] && [ ${var_hybrid_mode} = false ] && [ ${var_offline_mode} = false ] && [ -z ${EGG_MODE} ] && [ ${var_no_otp} = false ]; then cp_print "Please enter OTP token []:" ${FORCE_STDOUT} read -r var_token + attempts=0 + max_attempts=3 while [ -z "$var_token" ]; do + attempts=$((attempts + 1)) + if [ "$attempts" -gt "$max_attempts" ]; then + cp_print "Maximum attempts exceeded. open-appsec Nano Agent registration failed. Failed to get token" ${FORCE_STDOUT} + exit 1 + fi cp_print "You must enter OTP token[]:" ${FORCE_STDOUT} read -r var_token done @@ -1201,7 +1214,7 @@ install_orchestration() execution_flags="execution_flags=\"--orchestration-mode=${var_orchestration_mode}\"" echo $execution_flags >> ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg - if [ $var_arch = "gaia" -o "$is_smb" = "1" ]; then + if [ $var_arch = "gaia" -o $var_arch = "gaia_arm" -o "$is_smb" = "1" ]; then sed -i '1i gaia_ld_path='"$LD_LIBRARY_PATH"'' ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg fi