github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Mar 21st 2024 update

Browse Source
This commit is contained in:
Ned Wright
2024-03-21 15:31:38 +00:00
parent 0d22790ebe
commit c20fa9f966
100 changed files with 3851 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/intelligence_is_v2/include/intelligence_server.h
View File

@@ -37,7 +37,7 @@ private:
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;
bool is_local_intelligence;
bool is_local_intelligence = false;
Maybe<std::string> server_ip = genError("No server ip set");
Maybe<unsigned int> server_port = genError("No port is set");
I_Messaging * i_message = nullptr;

20
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -244,7 +244,12 @@ public:
bool
sendInvalidation(const Invalidation &invalidation) const override
{
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
if (hasLocalInvalidationSupport()) {
return sendLocalInvalidation(invalidation);
}
else {
return sendGlobalInvalidation(invalidation);
}
}
Maybe<uint>
@@ -287,9 +292,14 @@ public:
private:
bool
hasLocalIntelligence() const
hasLocalInvalidationSupport() const
{
return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
auto is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
if (!is_supported) {
is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.supportInvalidation");
}
return is_supported;
}
bool
@@ -330,6 +340,7 @@ private:
MessageMetadata invalidation_req_md(server, *port);
invalidation_req_md.insertHeaders(getHTTPHeaders());
invalidation_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
invalidation_uri,
@@ -411,6 +422,7 @@ private:
dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
MessageMetadata registration_req_md(server, *port);
registration_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
registration_uri,
@@ -423,7 +435,7 @@ private:
void
sendReccurringInvalidationRegistration() const
{
if (!hasLocalIntelligence() || invalidations.empty()) return;
if (!hasLocalInvalidationSupport() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}

5
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@@ -411,8 +411,9 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
"}\n"
);
MessageMetadata md;
EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str)));
).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
@@ -421,6 +422,7 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
auto iter = vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
@@ -490,6 +492,7 @@ TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
EXPECT_TRUE(maybe_ans.ok());
EXPECT_EQ(md.getHostName(), "127.0.0.1");
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)

110
core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
View File

@@ -25,59 +25,106 @@ TEST(InvalidationBasic, SettersAndGetters)
EXPECT_EQ(invalidation.getClassifier(ClassifierType::GROUP), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::ORDER), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), "");
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD);
EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok());
EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok());
EXPECT_FALSE(invalidation.getStringAttr("attr1").ok());
EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok());
EXPECT_FALSE(invalidation.getSourceId().ok());
EXPECT_FALSE(invalidation.getObjectType().ok());
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
invalidation
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc");
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals);
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4");
EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(invalidation.getSourceId().unpack(), "id");
EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET);
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE);
}
TEST(InvalidationBasic, Matching)
{
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
auto base_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals);
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false);
auto matching_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::ADD);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
auto not_matching_invalidation_type = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type));
auto missing_attr_invalidation_main = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringAttr("main_attr2", "2")
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main));
auto missing_attr_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringAttr("attr2", "2", false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -88,9 +135,12 @@ TEST(InvalidationBasic, Matching)
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringSetAttr("attr1", vals2)
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringSetAttr("main_attr1", vals2)
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -170,17 +220,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -192,16 +245,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, sending_private_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -221,11 +278,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -237,10 +296,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, register_for_invalidation)
@@ -257,19 +319,23 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
set<string> vals = { "11", "55", "22" };
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringSetAttr("attr3", vals, false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string body;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillOnce(DoAll(
SaveArg<2>(&body),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -278,6 +344,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\""));
EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\""));
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]"));
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, invalidation_callback)
@@ -321,7 +389,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1);
EXPECT_EQ(recieved_invalidations[0].getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals);
}
TEST_F(IntelligenceInvalidation, delete_invalidation_callback)

5
core/intelligence_is_v2/intelligence_server.cc
View File

@@ -43,7 +43,7 @@ Sender::Sender(IntelligenceRequest request) : request(request)
}
auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
if (setting_server_ip.ok()) server_ip = *setting_server_ip;
if (setting_server_ip.ok() && is_local_intelligence) server_ip = *setting_server_ip;
}
Maybe<Response>
@@ -80,6 +80,7 @@ Sender::sendQueryObjectToLocalServer(bool is_primary_port)
server_port = *local_port;
conn_flags.reset();
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
auto res = sendQueryMessage();
@@ -173,7 +174,7 @@ Sender::sendMessage()
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE,
req_md

103
core/intelligence_is_v2/invalidation.cc
View File

@@ -37,16 +37,16 @@ Invalidation::setClassifier(ClassifierType type, const string &val)
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val)
Invalidation::setStringAttr(const string &attr, const string &val, bool is_main)
{
string_main_attr[attr] = val;
is_main ? string_main_attr[attr] = val : string_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
Invalidation::setStringSetAttr(const string &attr, const set<string> &val, bool is_main)
{
set_string_main_attr[attr] = val;
is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val;
return *this;
}
@@ -64,8 +64,15 @@ Invalidation::setObjectType(ObjectType type)
return *this;
}
Invalidation &
Invalidation::setInvalidationType(InvalidationType type)
{
invalidation_type = type;
return *this;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
Invalidation::getStringMainAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
@@ -73,13 +80,29 @@ Invalidation::getStringAttr(const string &attr) const
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
Invalidation::getStringSetMainAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_attr.find(attr);
if (val_ref == string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_attr.find(attr);
if (val_ref == set_string_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
@@ -110,6 +133,12 @@ static const map<Intelligence::ObjectType, string> convertObjectType = {
{ Intelligence::ObjectType::SHORTLIVED, "shortLived" }
};
static const map<Intelligence::InvalidationType, string> convertInvalidationType = {
{ Intelligence::InvalidationType::ADD, "add" },
{ Intelligence::InvalidationType::DELETE, "delete" },
{ Intelligence::InvalidationType::UPDATE, "update" }
};
Maybe<string>
Invalidation::genJson() const
{
@@ -145,6 +174,7 @@ Invalidation::genObject() const
}
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
@@ -161,9 +191,35 @@ Invalidation::genObject() const
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &val : attr.second) {
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << val << "\"";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
if (!string_attr.empty() || !set_string_attr.empty()) {
invalidation << ", \"attributes\": [ ";
bool first = true;
for (auto &attr : string_attr) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
@@ -208,16 +264,29 @@ Invalidation::matches(const Invalidation &other) const
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (invalidation_type != other.invalidation_type) return false;
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
if (!other.hasMainAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasMainAttr(key_values.first, value)) return false;
}
}
for (auto &key_value : string_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
@@ -227,7 +296,7 @@ Invalidation::matches(const Invalidation &other) const
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
Invalidation::hasMainAttr(const string &key, const string &value) const
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
@@ -239,3 +308,17 @@ Invalidation::hasAttr(const string &key, const string &value) const
return false;
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_attr.find(key);
if (string_elem != string_attr.end()) return string_elem->second == value;
auto set_string_elem = set_string_attr.find(key);
if (set_string_elem != set_string_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
}
return false;
}