github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2026-01-17 16:00:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jan 06 2026 dev (#387)

Browse Source 
* sync code

* update code to support brotli

* update code to support brotli

* update code to support brotli

* sync code

* fix findBrotli

* sync code

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
This commit is contained in:
Daniel-Eisenberg
2026-01-13 17:17:52 +02:00
committed by GitHub
parent c1058db57d
commit e7b6e51b31
216 changed files with 12601 additions and 2825 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc
View File

@@ -20,30 +20,60 @@ USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
TrustedSourcesConfidenceCalculator::TrustedSourcesConfidenceCalculator(
std::string path,
const std::string& remotePath,
const std::string& assetId)
const std::string &remotePath,
const std::string &assetId)
:
SerializeToLocalAndRemoteSyncBase(std::chrono::minutes(120),
SYNC_WAIT_TIME,
path,
(remotePath == "") ? remotePath : remotePath + "/Trust",
assetId,
"TrustedSourcesConfidenceCalculator")
"TrustedSourcesConfidenceCalculator"),
m_persistent_state(),
m_incremental_logger(std::make_shared<KeyValSourceLogger>())
{
restore();
}
bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t minSources) const
{
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state first (accumulated data from previous syncs)
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
auto sourceSetItr = sourceCtrItr->second.find(value);
if (sourceSetItr != sourceCtrItr->second.end())
{
size_t persistent_sources = sourceSetItr->second.size();
if (persistent_sources >= minSources) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << persistent_sources << " (persistent only)";
return true;
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr->second.find(src) == sourceSetItr->second.end()) {
incremental_sources++;
}
}
}
}
}
size_t total_sources = persistent_sources + incremental_sources;
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << sourceSetItr->second.size();
return sourceSetItr->second.size() >= minSources;
<< " : " << value << " is " << total_sources << " (persistent: " << persistent_sources
<< ", incremental: " << incremental_sources << ")";
return total_sources >= minSources;
}
else
{
@@ -52,6 +82,18 @@ bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t
}
else
{
// Check if data exists only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << incr_set_itr->second.size() << " (incremental only)";
return incr_set_itr->second.size() >= minSources;
}
}
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return false;
@@ -76,25 +118,36 @@ private:
S2C_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger)
};
class TrsutedSourcesLogger : public RestGetFile
class TrustedSourcesLogger : public RestGetFile
{
public:
TrsutedSourcesLogger(const TrustedSourcesConfidenceCalculator::KeyValSourceLogger& _logger)
: logger(_logger)
TrustedSourcesLogger(std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> _logger_ptr)
: logger_ptr(_logger_ptr)
{
logger = move(*logger_ptr);
}
private:
std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> logger_ptr;
C2S_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger);
};
bool TrustedSourcesConfidenceCalculator::postData()
{
if (m_incremental_logger->empty())
{
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping";
return true; // Nothing to post
}
std::string url = getPostDataUrl();
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url;
mergeIncrementalToPersistent();
TrustedSourcesLogger logger(m_incremental_logger);
// Clear and reset incremental logger for next cycle
m_incremental_logger = std::make_shared<KeyValSourceLogger>();
TrsutedSourcesLogger logger(m_logger);
bool ok = sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
@@ -104,12 +157,12 @@ bool TrustedSourcesConfidenceCalculator::postData()
return ok;
}
void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>& files)
void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string> &files)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the window data for trusted sources";
std::string url = getPostDataUrl();
std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
for (auto file : files)
for (const auto &file : files) // Use const reference to avoid copying
{
if (file == sentFile)
{
@@ -135,19 +188,22 @@ void TrustedSourcesConfidenceCalculator::processData()
{
}
void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::string>& files)
void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::string> &files)
{
m_logger.clear();
pullProcessedData(files);
}
void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string>& files) {
Maybe<std::string> TrustedSourcesConfidenceCalculator::getRemoteStateFilePath() const
{
return m_remotePath + "/remote/data.data";
}
void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string> &files) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources";
bool pull_ok = false;
for (auto file: files) {
for (const auto &file : files) { // Use const reference
GetTrustedFile getTrustFile;
bool res = sendObjectWithRetry(getTrustFile,
bool res = sendObject(getTrustFile,
HTTPMethod::GET,
getUri() + "/" + file);
pull_ok |= res;
@@ -165,43 +221,89 @@ void TrustedSourcesConfidenceCalculator::postProcessedData()
std::string url = getUri() + "/" + m_remotePath + "/processed/data.data";
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the processed data to: " << url;
TrsutedSourcesLogger logger(m_logger);
// Send persistent state as processed data
auto logger_ptr = std::make_shared<TrustedSourcesConfidenceCalculator::KeyValSourceLogger>(m_persistent_state);
TrustedSourcesLogger logger(logger_ptr);
sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
}
TrustedSourcesConfidenceCalculator::ValuesSet TrustedSourcesConfidenceCalculator::getConfidenceValues(
const Key& key,
const Key &key,
size_t minSources) const
{
ValuesSet values;
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
for (auto sourceSetItr : sourceCtrItr->second)
{
if (sourceSetItr.second.size() >= minSources)
size_t persistent_sources = sourceSetItr.second.size();
if (persistent_sources >= minSources)
{
values.insert(sourceSetItr.first);
continue; // No need to check incremental logger if we already have enough sources
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(sourceSetItr.first);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr.second.find(src) == sourceSetItr.second.end()) {
incremental_sources++;
}
}
}
}
}
if (persistent_sources + incremental_sources >= minSources)
{
values.insert(sourceSetItr.first);
}
}
}
else
{
// Also check values that exist only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
for (auto incr_set_itr : incr_ctr_itr->second) {
// Skip if already processed in persistent state
if (sourceCtrItr != m_persistent_state.end() &&
sourceCtrItr->second.find(incr_set_itr.first) != sourceCtrItr->second.end()) {
continue;
}
if (incr_set_itr.second.size() >= minSources) {
values.insert(incr_set_itr.first);
}
}
}
}
if (values.empty()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return values;
}
void TrustedSourcesConfidenceCalculator::serialize(std::ostream& stream)
void TrustedSourcesConfidenceCalculator::serialize(std::ostream &stream)
{
cereal::JSONOutputArchive archive(stream);
archive(cereal::make_nvp("version", 2), cereal::make_nvp("logger", m_logger));
archive(cereal::make_nvp("version", 3), cereal::make_nvp("persistent_state", m_persistent_state));
}
void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
void TrustedSourcesConfidenceCalculator::deserialize(std::istream &stream)
{
cereal::JSONInputArchive archive(stream);
size_t version = 0;
@@ -218,24 +320,31 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
switch (version)
{
case 3:
{
archive(cereal::make_nvp("persistent_state", m_persistent_state));
break;
}
case 2:
{
archive(cereal::make_nvp("logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("logger", m_persistent_state));
break;
}
case 1:
{
KeyValSourceLogger logger;
archive(cereal::make_nvp("logger", logger));
for (auto& log : logger)
for (auto &log : logger)
{
m_logger[normalize_param(log.first)] = log.second;
m_persistent_state[normalize_param(log.first)] = log.second;
}
break;
}
case 0:
{
archive(cereal::make_nvp("m_logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("m_logger", m_persistent_state));
break;
}
default:
@@ -244,17 +353,17 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
}
}
void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger& logs)
void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger &logs)
{
for (auto& srcCounterItr : logs)
for (auto &srcCounterItr : logs)
{
for (auto& sourcesItr : srcCounterItr.second)
for (auto &sourcesItr : srcCounterItr.second)
{
for (auto& src : sourcesItr.second)
for (auto &src : sourcesItr.second)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Registering the source: " << src
<< " for the value: " << sourcesItr.first << " and the key: " << srcCounterItr.first;
m_logger[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
m_persistent_state[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
}
}
}
@@ -269,10 +378,28 @@ void TrustedSourcesConfidenceCalculator::log(Key key, Val value, Source source)
<< key
<< " from the source: "
<< source;
m_logger[key][value].insert(source);
(*m_incremental_logger)[key][value].insert(source);
}
void TrustedSourcesConfidenceCalculator::reset()
{
m_logger.clear();
m_persistent_state.clear();
m_incremental_logger->clear();
}
void TrustedSourcesConfidenceCalculator::mergeIncrementalToPersistent()
{
// Merge incremental data into persistent state (same logic as in postData but without network operations)
for (const auto &keyEntry : *m_incremental_logger) {
const std::string &key = keyEntry.first;
for (const auto &valueEntry : keyEntry.second) {
const std::string &value = valueEntry.first;
for (const std::string &source : valueEntry.second) {
m_persistent_state[key][value].insert(source);
}
}
}
// Clear incremental logger after merging
m_incremental_logger->clear();
}