github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 11:16:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Mar 21st 2024 update

Browse Source
This commit is contained in:
Ned Wright
2024-03-21 15:31:38 +00:00
parent 0d22790ebe
commit c20fa9f966
100 changed files with 3851 additions and 453 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc
View File

@@ -95,6 +95,7 @@ TEST_F(AgentReporterTest, dataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
AgentDataReport() << AgentReportField(custom_data);;
@@ -112,6 +113,7 @@ TEST_F(AgentReporterTest, labeledDataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
AgentDataReport() << AgentReportFieldWithLabel("this_is_custom_label", data);
@@ -131,6 +133,7 @@ TEST_F(AgentReporterTest, multiDataReport)
" }"
"\n}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -158,6 +161,7 @@ TEST_F(AgentReporterTest, multiDataReportWithRegistrationData)
" \"architecture\": \"aaa\"\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -181,6 +185,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" \"additionalMetaData\": {}\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -200,6 +205,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -219,6 +225,7 @@ TEST_F(AgentReporterTest, basicAttrTest)
" \"additionalMetaData\": {}\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -247,6 +254,7 @@ TEST_F(AgentReporterTest, advancedAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -272,6 +280,7 @@ TEST_F(AgentReporterTest, advancedAttrTest)
" }\n"
"}",
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -297,6 +306,7 @@ TEST_F(AgentReporterTest, RestDetailsTest)
"/agents",
rest_call_parameters.str(),
MessageCategory::GENERIC,
_,
_
)).Times(1);
@@ -366,6 +376,7 @@ TEST_F(AgentReporterTest, PersistenceAttrTest)
"/agents",
expected_attributes,
MessageCategory::GENERIC,
_,
_
)).Times(1);
EXPECT_TRUE(report->sendAttributes());

3
core/debug_is/debug_is_ut/debug_ut.cc
View File

@@ -824,6 +824,7 @@ TEST(DebugFogTest, fog_stream)
"/api/v1/agents/events/bulk",
_,
_,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -980,7 +981,7 @@ TEST(DebugFogTest, fog_stream)
.WillOnce(DoAll(InvokeMainLoopCB(), Return(0)));
string message_body_1, message_body_2;
EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _))
EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _, _))
.WillOnce(SaveArg<2>(&message_body_1))
.WillOnce(SaveArg<2>(&message_body_2));

13
core/include/general/common.h
View File

@@ -27,6 +27,7 @@
#include <memory>
#include <string>
#include <vector>
#include <sstream>
#include <iomanip>
#include <sys/types.h>
@@ -132,6 +133,18 @@ operator<<(ostream &os, const pair<const First, Second> &printable_pair)
return os;
}
template <typename Printable>
ostream &
operator<<(ostream &os, const vector<Printable> &obj)
{
bool first = true;
for (const auto &val : obj) {
os << (first ? "" : ", ") << val;
first = false;
}
return os;
}
} // namespace std
#endif // __COMMON_H__

6
core/include/services_sdk/interfaces/i_messaging.h
View File

@@ -56,14 +56,16 @@ public:
const std::string &uri,
serializableObject &req_obj,
const MessageCategory category = MessageCategory::GENERIC,
MessageMetadata message_metadata = MessageMetadata());
MessageMetadata message_metadata = MessageMetadata(),
bool force_buffering = true);
virtual void sendAsyncMessage(
const HTTPMethod method,
const std::string &uri,
const std::string &body,
const MessageCategory category,
MessageMetadata message_metadata
const MessageMetadata &message_metadata = MessageMetadata(),
bool force_buffering = true
) = 0;
virtual Maybe<HTTPResponse, HTTPResponse> sendSyncMessage(

2
core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h
View File

@@ -28,6 +28,7 @@ AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
SerializableMultiMap<std::string, std::vector<std::string>> tmp_multimap;
ar(
cereal::make_nvp("schemaVersion", asset_schema_version),
cereal::make_nvp("assetType", asset_type),
cereal::make_nvp("assetTypeSchemaVersion", asset_type_schema_version),
cereal::make_nvp("class", asset_class),
cereal::make_nvp("category", asset_category),
@@ -65,7 +66,6 @@ AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
} catch(...) {}
ar(cereal::make_nvp("sources", sources));
ar(cereal::make_nvp("assetType", asset_type));
}
template <typename UserSerializableReplyAttr>

6
core/include/services_sdk/interfaces/messaging/interface_impl.h
View File

@@ -128,7 +128,8 @@ I_Messaging::sendAsyncMessage(
const std::string &uri,
serializableObject &req_obj,
const MessageCategory category,
MessageMetadata message_metadata)
MessageMetadata message_metadata,
bool force_buffering)
{
Maybe<std::string> req_body = req_obj.genJson();
if (!req_body.ok()) {
@@ -141,7 +142,8 @@ I_Messaging::sendAsyncMessage(
uri,
req_body.unpack(),
category,
message_metadata
message_metadata,
force_buffering
);
}

5
core/include/services_sdk/interfaces/mock/mock_messaging.h
View File

@@ -18,14 +18,15 @@ public:
MessageMetadata
)
);
MOCK_METHOD5(
MOCK_METHOD6(
sendAsyncMessage,
void (
HTTPMethod,
const string &,
const string &,
MessageCategory,
MessageMetadata
const MessageMetadata &,
bool
)
);

2
core/include/services_sdk/resources/debug_flags.h
View File

@@ -80,6 +80,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP)
DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP)
DEFINE_FLAG(D_WAAP_HEADERS, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_HTML, D_WAAP_PARSER)
@@ -96,6 +97,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP)
DEFINE_FLAG(D_IPS, D_COMPONENT)

4
core/include/services_sdk/resources/generic_metric.h
View File

@@ -58,7 +58,8 @@ public:
const ReportIS::IssuingEngine &_issuing_engine,
std::chrono::seconds _report_interval,
bool _reset,
ReportIS::Audience _audience = ReportIS::Audience::INTERNAL
ReportIS::Audience _audience = ReportIS::Audience::INTERNAL,
bool _force_buffering = false
);
template <typename Value>
@@ -107,6 +108,7 @@ private:
std::chrono::seconds report_interval;
std::vector<MetricCalc *> calcs;
bool reset;
bool force_buffering = false;
Context ctx;
};

13
core/include/services_sdk/resources/intelligence_invalidation.h
View File

@@ -29,6 +29,7 @@ namespace Intelligence
enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED };
enum class InvalidationType { ADD, DELETE, UPDATE };
class Invalidation
{
@@ -36,16 +37,20 @@ public:
Invalidation(const std::string &class_value);
Invalidation & setClassifier(ClassifierType type, const std::string &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val, bool is_main = true);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val, bool is_main = true);
Invalidation & setSourceId(const std::string &id);
Invalidation & setObjectType(ObjectType type);
Invalidation & setInvalidationType(InvalidationType type);
std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
Maybe<std::string, void> getStringMainAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetMainAttr(const std::string &attr) const;
Maybe<std::string, void> getStringAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
const Maybe<std::string, void> & getSourceId() const { return source_id; }
const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
InvalidationType getInvalidationType() const { return invalidation_type; }
bool report(I_Intelligence_IS_V2 *interface) const;
@@ -59,13 +64,17 @@ public:
bool matches(const Invalidation &other) const;
private:
bool hasMainAttr(const std::string &key, const std::string &value) const;
bool hasAttr(const std::string &key, const std::string &value) const;
EnumArray<ClassifierType, std::string, 6> classifiers;
std::map<std::string, std::string> string_main_attr;
std::map<std::string, std::set<std::string>> set_string_main_attr;
std::map<std::string, std::string> string_attr;
std::map<std::string, std::set<std::string>> set_string_attr;
Maybe<std::string, void> source_id;
Maybe<ObjectType, void> object_type;
InvalidationType invalidation_type = InvalidationType::ADD;
Maybe<uint, void> listening_id;
};

2
core/intelligence_is_v2/include/intelligence_server.h
View File

@@ -37,7 +37,7 @@ private:
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;
bool is_local_intelligence;
bool is_local_intelligence = false;
Maybe<std::string> server_ip = genError("No server ip set");
Maybe<unsigned int> server_port = genError("No port is set");
I_Messaging * i_message = nullptr;

20
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -244,7 +244,12 @@ public:
bool
sendInvalidation(const Invalidation &invalidation) const override
{
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
if (hasLocalInvalidationSupport()) {
return sendLocalInvalidation(invalidation);
}
else {
return sendGlobalInvalidation(invalidation);
}
}
Maybe<uint>
@@ -287,9 +292,14 @@ public:
private:
bool
hasLocalIntelligence() const
hasLocalInvalidationSupport() const
{
return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
auto is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
if (!is_supported) {
is_supported = getProfileAgentSettingWithDefault<bool>(false, "agent.config.supportInvalidation");
}
return is_supported;
}
bool
@@ -330,6 +340,7 @@ private:
MessageMetadata invalidation_req_md(server, *port);
invalidation_req_md.insertHeaders(getHTTPHeaders());
invalidation_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
invalidation_uri,
@@ -411,6 +422,7 @@ private:
dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
MessageMetadata registration_req_md(server, *port);
registration_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
return message->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
registration_uri,
@@ -423,7 +435,7 @@ private:
void
sendReccurringInvalidationRegistration() const
{
if (!hasLocalIntelligence() || invalidations.empty()) return;
if (!hasLocalInvalidationSupport() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}

5
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@@ -411,8 +411,9 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
"}\n"
);
MessageMetadata md;
EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str)));
).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
@@ -421,6 +422,7 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
auto iter = vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
@@ -490,6 +492,7 @@ TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
EXPECT_TRUE(maybe_ans.ok());
EXPECT_EQ(md.getHostName(), "127.0.0.1");
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)

110
core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
View File

@@ -25,59 +25,106 @@ TEST(InvalidationBasic, SettersAndGetters)
EXPECT_EQ(invalidation.getClassifier(ClassifierType::GROUP), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::ORDER), "");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), "");
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD);
EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok());
EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok());
EXPECT_FALSE(invalidation.getStringAttr("attr1").ok());
EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok());
EXPECT_FALSE(invalidation.getSourceId().ok());
EXPECT_FALSE(invalidation.getObjectType().ok());
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
invalidation
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc");
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals);
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4");
EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(invalidation.getSourceId().unpack(), "id");
EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET);
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE);
}
TEST(InvalidationBasic, Matching)
{
set<string> vals = { "2", "3" };
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
auto base_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals);
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false);
auto matching_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::ADD);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
auto not_matching_invalidation_type = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type));
auto missing_attr_invalidation_main = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringAttr("main_attr2", "2")
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main));
auto missing_attr_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("attr1", "1")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "6")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringAttr("attr2", "2", false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -88,9 +135,12 @@ TEST(InvalidationBasic, Matching)
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringSetAttr("attr1", vals2)
.setStringSetAttr("attr2", vals)
.setStringAttr("attr3", "6")
.setStringSetAttr("main_attr1", vals2)
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -170,17 +220,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -192,16 +245,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, sending_private_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -221,11 +278,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
string invalidation_json;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -237,10 +296,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" } ]"
"\"mainAttributes\": [ { \"attr2\": \"2\" } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, register_for_invalidation)
@@ -257,19 +319,23 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
set<string> vals = { "11", "55", "22" };
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringSetAttr("attr3", vals, false)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string body;
MessageMetadata md;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillOnce(DoAll(
SaveArg<2>(&body),
SaveArg<4>(&md),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
@@ -278,6 +344,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\""));
EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\""));
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]"));
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, invalidation_callback)
@@ -321,7 +389,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1);
EXPECT_EQ(recieved_invalidations[0].getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals);
}
TEST_F(IntelligenceInvalidation, delete_invalidation_callback)

5
core/intelligence_is_v2/intelligence_server.cc
View File

@@ -43,7 +43,7 @@ Sender::Sender(IntelligenceRequest request) : request(request)
}
auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
if (setting_server_ip.ok()) server_ip = *setting_server_ip;
if (setting_server_ip.ok() && is_local_intelligence) server_ip = *setting_server_ip;
}
Maybe<Response>
@@ -80,6 +80,7 @@ Sender::sendQueryObjectToLocalServer(bool is_primary_port)
server_port = *local_port;
conn_flags.reset();
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
auto res = sendQueryMessage();
@@ -173,7 +174,7 @@ Sender::sendMessage()
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE,
req_md

103
core/intelligence_is_v2/invalidation.cc
View File

@@ -37,16 +37,16 @@ Invalidation::setClassifier(ClassifierType type, const string &val)
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val)
Invalidation::setStringAttr(const string &attr, const string &val, bool is_main)
{
string_main_attr[attr] = val;
is_main ? string_main_attr[attr] = val : string_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
Invalidation::setStringSetAttr(const string &attr, const set<string> &val, bool is_main)
{
set_string_main_attr[attr] = val;
is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val;
return *this;
}
@@ -64,8 +64,15 @@ Invalidation::setObjectType(ObjectType type)
return *this;
}
Invalidation &
Invalidation::setInvalidationType(InvalidationType type)
{
invalidation_type = type;
return *this;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
Invalidation::getStringMainAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
@@ -73,13 +80,29 @@ Invalidation::getStringAttr(const string &attr) const
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
Invalidation::getStringSetMainAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_attr.find(attr);
if (val_ref == string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_attr.find(attr);
if (val_ref == set_string_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
@@ -110,6 +133,12 @@ static const map<Intelligence::ObjectType, string> convertObjectType = {
{ Intelligence::ObjectType::SHORTLIVED, "shortLived" }
};
static const map<Intelligence::InvalidationType, string> convertInvalidationType = {
{ Intelligence::InvalidationType::ADD, "add" },
{ Intelligence::InvalidationType::DELETE, "delete" },
{ Intelligence::InvalidationType::UPDATE, "update" }
};
Maybe<string>
Invalidation::genJson() const
{
@@ -145,6 +174,7 @@ Invalidation::genObject() const
}
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
@@ -161,9 +191,35 @@ Invalidation::genObject() const
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &val : attr.second) {
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << val << "\"";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
if (!string_attr.empty() || !set_string_attr.empty()) {
invalidation << ", \"attributes\": [ ";
bool first = true;
for (auto &attr : string_attr) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
@@ -208,16 +264,29 @@ Invalidation::matches(const Invalidation &other) const
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (invalidation_type != other.invalidation_type) return false;
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
if (!other.hasMainAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasMainAttr(key_values.first, value)) return false;
}
}
for (auto &key_value : string_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
@@ -227,7 +296,7 @@ Invalidation::matches(const Invalidation &other) const
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
Invalidation::hasMainAttr(const string &key, const string &value) const
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
@@ -239,3 +308,17 @@ Invalidation::hasAttr(const string &key, const string &value) const
return false;
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_attr.find(key);
if (string_elem != string_attr.end()) return string_elem->second == value;
auto set_string_elem = set_string_attr.find(key);
if (set_string_elem != set_string_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
}
return false;
}

3
core/logging/k8s_svc_stream.cc
View File

@@ -41,6 +41,8 @@ K8sSvcStream::sendLog(const Report &log)
MessageMetadata rest_req_md(svc_host, 80);
rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId());
rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
bool ok = i_msg->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
K8sSvc_log_uri,
@@ -69,6 +71,7 @@ K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only)
MessageMetadata rest_req_md(svc_host, 80);
rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getTenantId());
rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
bool ok = i_msg->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
K8sSvc_log_uri,

6
core/logging/logging_ut/logging_ut.cc
View File

@@ -171,6 +171,7 @@ public:
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&body));
}
@@ -745,6 +746,7 @@ TEST_F(LogTest, FogBulkLogs)
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&local_body));
@@ -1143,6 +1145,7 @@ TEST(LogTestInstanceAwareness, LogGenInstanceAwareness)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@@ -1248,6 +1251,7 @@ TEST(LogTestWithoutComponent, RegisterBasicConfig)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@@ -1295,6 +1299,7 @@ TEST(LogTestWithoutComponent, RegisterAdvancedConfig)
_,
_,
MessageCategory::LOG,
_,
_
)).Times(AnyNumber());
@@ -1366,6 +1371,7 @@ TEST_F(LogTest, BulkModification)
_,
_,
MessageCategory::LOG,
_,
_
)).WillRepeatedly(SaveArg<2>(&local_body));

2
core/mainloop/mainloop_ut/mainloop_ut.cc
View File

@@ -51,7 +51,7 @@ public:
{
EXPECT_CALL(
mock_msg,
sendAsyncMessage(_, "/api/v1/agents/events", _, _, _)
sendAsyncMessage(_, "/api/v1/agents/events", _, _, _, _)
).Times(2).WillRepeatedly(
WithArgs<2, 3>(
Invoke(

79
core/messaging/connection/connection.cc
View File

@@ -38,6 +38,10 @@ USE_DEBUG_FLAG(D_CONNECTION);
static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time");
static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time");
static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response");
static const HTTPResponse close_error(
HTTPStatusCode::HTTP_UNKNOWN,
"The previous request failed to receive a response. Closing the connection"
);
const string &
MessageConnectionKey::getHostName() const
@@ -125,6 +129,12 @@ public:
return key;
}
bool
shouldCloseConnection() const
{
return should_close_connection;
}
bool
isOverProxy() const
{
@@ -152,12 +162,12 @@ public:
if (establishConnection().ok()) {
dbgDebug(D_MESSAGING) << "Reestablish connection";
return true;
return false;
}
dbgWarning(D_MESSAGING) << "Reestablish connection failed";
active = genError(curr_time + chrono::seconds(300));
return false;
return true;
}
Maybe<void>
@@ -200,6 +210,7 @@ public:
<< key.getPort()
<< (isOverProxy() ? ", Over proxy: " + settings.getProxyHost() + ":" + to_string(key.getPort()) : "");
active = Maybe<void, chrono::seconds>();
should_close_connection = false;
return Maybe<void>();
}
@@ -443,25 +454,40 @@ private:
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
auto bio_connect = BIO_do_connect(bio.get());
uint attempts_count = 0;
while (i_time->getMonotonicTime() < conn_end_time) {
attempts_count++;
if (BIO_do_connect(bio.get()) > 0) {
if (isUnsecure() || isOverProxy()) return Maybe<void>();
return performHandshakeAndVerifyCert(i_time, i_mainloop);
}
auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
while (i_time->getMonotonicTime() < conn_end_time && bio_connect <= 0) {
if (!BIO_should_retry(bio.get())) {
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
string bio_error = ERR_error_string(ERR_get_error(), nullptr);
return genError("Failed to connect (BIO won't retry!): " + bio_error);
return genError(
"Failed to connect to: " +
full_address +
", error: " +
bio_error +
". Connection suspended for 60 seconds");
}
if ((attempts_count % 10) == 0) i_mainloop->yield(true);
attempts_count++;
if (!isBioSocketReady()) {
i_mainloop->yield((attempts_count % 10) == 0);
continue;
}
bio_connect = BIO_do_connect(bio.get());
}
return genError("Failed to establish new connection to " + full_address + " after reaching timeout.");
if (bio_connect > 0) {
if (isUnsecure() || isOverProxy()) return Maybe<void>();
return performHandshakeAndVerifyCert(i_time, i_mainloop);
}
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
return genError(
"Failed to establish new connection to: " +
full_address +
" after reaching timeout." +
" Connection suspended for 60 seconds");
}
Maybe<uint, HTTPResponse>
@@ -544,6 +570,7 @@ private:
Maybe<HTTPResponse, HTTPResponse>
sendAndReceiveData(const string &request, bool is_connect)
{
dbgFlow(D_CONNECTION) << "Sending and receiving data";
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
while (lock) {
i_mainloop->yield(true);
@@ -551,6 +578,11 @@ private:
lock = true;
auto unlock = make_scope_exit([&] () { lock = false; });
if (should_close_connection) {
dbgWarning(D_CONNECTION) << close_error.getBody();
return genError(close_error);
}
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
size_t data_left_to_send = request.length();
@@ -567,9 +599,15 @@ private:
HTTPResponseParser http_parser;
dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response";
while (!http_parser.hasReachedError()) {
if (i_time->getMonotonicTime() > receiving_end_time) return genError(receving_timeout);
if (i_time->getMonotonicTime() > receiving_end_time) {
should_close_connection = true;
return genError(receving_timeout);
};
auto receieved = receiveData();
if (!receieved.ok()) return receieved.passErr();
if (!receieved.ok()) {
should_close_connection = true;
return receieved.passErr();
}
auto response = http_parser.parseData(*receieved, is_connect);
if (response.ok()) {
dbgTrace(D_MESSAGING) << printOut(response.unpack().toString());
@@ -611,6 +649,7 @@ private:
uint failed_attempts = 0;
bool lock = false;
bool should_close_connection = false;
};
Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)
@@ -663,6 +702,12 @@ Connection::getConnKey() const
return pimpl->getConnKey();
}
bool
Connection::shouldCloseConnection() const
{
return pimpl->shouldCloseConnection();
}
bool
Connection::isOverProxy() const
{

6
core/messaging/connection/connection_comp.cc
View File

@@ -53,6 +53,10 @@ public:
{
auto conn = persistent_connections.find(MessageConnectionKey(host_name, port, category));
if (conn == persistent_connections.end()) return genError("No persistent connection found");
if (conn->second.shouldCloseConnection()) {
persistent_connections.erase(conn);
return genError("The connection needs to reestablish");
}
return conn->second;
}
@@ -93,6 +97,7 @@ private:
if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate);
auto connected = conn.establishConnection();
persistent_connections.emplace(conn_key, conn);
if (!connected.ok()) {
string connection_err = "Failed to establish connection. Error: " + connected.getErr();
@@ -101,7 +106,6 @@ private:
}
dbgTrace(D_CONNECTION) << "Connection establish succssesfuly";
persistent_connections.emplace(conn_key, conn);
return conn;
}

110
core/messaging/connection/connection_ut/connection_comp_ut.cc
View File

@@ -23,83 +23,13 @@
#include "mock/mock_encryptor.h"
#include "rest.h"
#include "rest_server.h"
#include "dummy_socket.h"
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_CONNECTION);
class DummySocket : Singleton::Consume<I_MainLoop>
{
public:
~DummySocket()
{
if (server_fd != -1) close(server_fd);
if (connection_fd != -1) close(connection_fd);
}
void
init()
{
server_fd = socket(AF_INET, SOCK_STREAM, 0);
dbgAssert(server_fd >= 0) << "Failed to open a socket";
int socket_enable = 1;
setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int));
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = htons(8080);
bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in));
listen(server_fd, 100);
}
void
acceptSocket()
{
if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr);
}
string
readFromSocket()
{
acceptSocket();
string res;
char buffer[1024];
while (int bytesRead = readRaw(buffer, sizeof(buffer))) {
res += string(buffer, bytesRead);
}
return res;
}
void
writeToSocket(const string &msg)
{
acceptSocket();
EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size());
}
private:
int
readRaw(char *buf, uint len)
{
struct pollfd s_poll;
s_poll.fd = connection_fd;
s_poll.events = POLLIN;
s_poll.revents = 0;
if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0;
return read(connection_fd, buf, len);
}
int server_fd = -1;
int connection_fd = -1;
};
static ostream &
operator<<(ostream &os, const BufferedMessage &)
{
@@ -179,7 +109,6 @@ TEST_F(TestConnectionComp, testEstablishNewConnection)
conn_metadata.setExternalCertificate("external cert");
auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
cout << "[OREN] read: " << endl;
ASSERT_TRUE(maybe_connection.ok());
auto get_conn = maybe_connection.unpack();
EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1");
@@ -228,6 +157,43 @@ TEST_F(TestConnectionComp, testSendRequest)
EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg);
}
TEST_F(TestConnectionComp, testCloseConnectionBeforeResponse)
{
// Create a connection
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags);
// Insert the connection to the map
auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
ASSERT_TRUE(maybe_connection.ok());
// Get the connection from the map - Should be successful
auto maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG);
ASSERT_TRUE(maybe_get_connection.ok());
auto conn = maybe_get_connection.unpack();
auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body");
ASSERT_TRUE(req.ok());
// force the connection to be closed
ON_CALL(mock_mainloop, yield(false)).WillByDefault(InvokeWithoutArgs([&] () { return; }));
EXPECT_CALL(mock_timer, getMonotonicTime())
.WillRepeatedly(Invoke([] () {static int j = 0; return chrono::microseconds(++j * 1000 * 1000);}));
auto maybe_response = i_conn->sendRequest(conn, *req);
ASSERT_TRUE(!maybe_response.ok());
ASSERT_EQ(
maybe_response.getErr().toString(),
"[Status-code]: -1 - HTTP_UNKNOWN, [Body]: Failed to receive all data in time"
);
auto maybe_get_closed_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG);
ASSERT_TRUE(!maybe_get_closed_connection.ok());
ASSERT_EQ(maybe_get_closed_connection.getErr(), "The connection needs to reestablish");
}
TEST_F(TestConnectionComp, testSendRequestReplyChunked)
{
Flags<MessageConnectionConfig> conn_flags;

1
core/messaging/include/connection.h
View File

@@ -63,6 +63,7 @@ public:
bool isOverProxy() const;
bool isUnsecure() const;
bool isSuspended();
bool shouldCloseConnection() const;
Maybe<void> establishConnection();
Maybe<HTTPResponse, HTTPResponse> sendRequest(const std::string &request);

100
core/messaging/include/dummy_socket.h Normal file
View File

@@ -0,0 +1,100 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __DUMMY_SOCKET_H__
#define __DUMMY_SOCKET_H__
#include <sys/socket.h>
#include <sys/types.h>
#include <arpa/inet.h>
#include <pthread.h>
#include <thread>
#include <fcntl.h>
#include <poll.h>
#include "singleton.h"
#include "i_mainloop.h"
#include "agent_core_utilities.h"
class DummySocket : Singleton::Consume<I_MainLoop>
{
public:
~DummySocket()
{
if (server_fd != -1) close(server_fd);
if (connection_fd != -1) close(connection_fd);
}
void
init()
{
server_fd = socket(AF_INET, SOCK_STREAM, 0);
dbgAssert(server_fd >= 0) << "Failed to open a socket";
int socket_enable = 1;
setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int));
struct sockaddr_in addr;
bzero(&addr, sizeof(addr));
addr.sin_family = AF_INET;
addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
addr.sin_port = htons(8080);
bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in));
listen(server_fd, 100);
}
void
acceptSocket()
{
if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr);
}
std::string
readFromSocket()
{
acceptSocket();
std::string res;
char buffer[1024];
while (int bytesRead = readRaw(buffer, sizeof(buffer))) {
res += std::string(buffer, bytesRead);
}
return res;
}
void
writeToSocket(const std::string &msg)
{
acceptSocket();
EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size());
}
private:
int
readRaw(char *buf, uint len)
{
struct pollfd s_poll;
s_poll.fd = connection_fd;
s_poll.events = POLLIN;
s_poll.revents = 0;
if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0;
return read(connection_fd, buf, len);
}
int server_fd = -1;
int connection_fd = -1;
};
#endif // __DUMMY_SOCKET_H__

3
core/messaging/include/messaging_comp.h
View File

@@ -50,7 +50,8 @@ public:
const std::string &uri,
const std::string &body,
MessageCategory category,
const MessageMetadata &message_metadata
const MessageMetadata &message_metadata,
bool force_buffering = true
);
Maybe<HTTPStatusCode, HTTPResponse> downloadFile(

5
core/messaging/messaging.cc
View File

@@ -55,10 +55,11 @@ public:
const std::string &uri,
const std::string &body,
const MessageCategory category,
MessageMetadata message_metadata
const MessageMetadata &message_metadata,
bool force_buffering
) override
{
return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata);
return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, force_buffering);
}
Maybe<HTTPStatusCode, HTTPResponse>

10
core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
View File

@@ -153,8 +153,6 @@ MessagingBufferComponent::Impl::pushNewBufferedMessage(
{
dbgTrace(D_MESSAGING_BUFFER) << "Pushing new message to buffer";
message_metadata.setShouldBufferMessage(false);
if (!force_immediate_writing) {
dbgDebug(D_MESSAGING_BUFFER) << "Holding message temporarily in memory";
memory_messages.emplace_back(body, method, uri, category, message_metadata);
@@ -296,12 +294,14 @@ MessagingBufferComponent::Impl::sendMessage()
HTTPStatusCode
MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) const
{
MessageMetadata message_metadata = message.getMessageMetadata();
message_metadata.setShouldBufferMessage(false);
auto res = messaging->sendSyncMessage(
message.getMethod(),
message.getURI(),
message.getBody(),
message.getCategory(),
message.getMessageMetadata()
message_metadata
);
if (res.ok()) return HTTPStatusCode::HTTP_OK;
@@ -316,7 +316,9 @@ MessagingBufferComponent::Impl::handleInMemoryMessages()
memory_messages.reserve(32);
for (const auto &message : messages) {
if (sendMessage(message) != HTTPStatusCode::HTTP_OK) writeToDisk(message);
if (sendMessage(message) != HTTPStatusCode::HTTP_OK) {
if (message.getMessageMetadata().shouldBufferMessage()) writeToDisk(message);
}
mainloop->yield();
}
}

22
core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc
View File

@@ -233,17 +233,36 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory)
string body_1 = "body1";
string body_2 = "body2";
string body_3 = "body3";
string body_4 = "body4";
string uri_1 = "uri_1";
string uri_2 = "uri_2";
string uri_3 = "uri_3";
string uri_4 = "uri_4";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata = MessageMetadata();
MessageMetadata msg_2_message_metadata = MessageMetadata();
msg_2_message_metadata.setShouldBufferMessage(true);
HTTPMethod method = HTTPMethod::POST;
buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(
body_2,
method,
uri_2,
category,
msg_2_message_metadata,
false
); // should be buffered
buffer_provider->pushNewBufferedMessage(body_3, method, uri_3, category, message_metadata, false);
buffer_provider->pushNewBufferedMessage(
body_4,
method,
uri_4,
category,
message_metadata,
false
); // shouldn't be buffered
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
Maybe<HTTPResponse, HTTPResponse> err = genError(res);
@@ -251,6 +270,7 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory)
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_1, body_1, _, _)).WillOnce(Return(res));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_2, body_2, _, _)).WillOnce(Return(err));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_3, body_3, _, _)).WillOnce(Return(res));
EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_4, body_4, _, _)).WillOnce(Return(err));
memory_routine();

7
core/messaging/messaging_comp/messaging_comp.cc
View File

@@ -136,10 +136,13 @@ MessagingComp::sendAsyncMessage(
const string &uri,
const string &body,
MessageCategory category,
const MessageMetadata &message_metadata
const MessageMetadata &message_metadata,
bool force_buffering
)
{
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
MessageMetadata new_message_metadata = message_metadata;
new_message_metadata.setShouldBufferMessage(force_buffering);
i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, new_message_metadata, false);
}
Maybe<HTTPStatusCode, HTTPResponse>

35
core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
View File

@@ -15,6 +15,7 @@
#include "mocks/mock_messaging_connection.h"
#include "rest.h"
#include "rest_server.h"
#include "dummy_socket.h"
using namespace std;
using namespace testing;
@@ -48,11 +49,13 @@ class TestMessagingComp : public testing::Test
public:
TestMessagingComp()
{
Debug::setUnitTestFlag(D_MESSAGING, Debug::DebugLevel::TRACE);
EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0)));
ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe<string>(fog_addr)));
ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe<uint16_t>(fog_port)));
messaging_comp.init();
dummy_socket.init();
}
void
@@ -70,8 +73,8 @@ public:
EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred")));
}
const string fog_addr = "1.2.3.4";
uint16_t fog_port = 80;
const string fog_addr = "127.0.0.1";
int fog_port = 8080;
CPTestTempfile agent_details_file;
MessagingComp messaging_comp;
::Environment env;
@@ -82,6 +85,7 @@ public:
NiceMock<MockTimeGet> mock_time_get;
NiceMock<MockAgentDetails> mock_agent_details;
NiceMock<MockProxyConfiguration> mock_proxy_conf;
DummySocket dummy_socket;
};
TEST_F(TestMessagingComp, testInitComp)
@@ -100,16 +104,19 @@ TEST_F(TestMessagingComp, testSendSyncMessage)
HTTPMethod method = HTTPMethod::POST;
string uri = "/test-uri";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "response!!");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata);
auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata);
ASSERT_TRUE(sending_res.ok());
HTTPResponse http_res = sending_res.unpack();
EXPECT_EQ(http_res.getBody(), "response!!");
@@ -126,7 +133,7 @@ TEST_F(TestMessagingComp, testSendAsyncMessage)
MessageMetadata message_metadata;
EXPECT_CALL(mock_messaging_buffer, pushNewBufferedMessage(body, method, uri, category, _, _)).Times(1);
messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata);
messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, true);
}
TEST_F(TestMessagingComp, testSendSyncMessageOnSuspendedConn)
@@ -163,16 +170,18 @@ TEST_F(TestMessagingComp, testUploadFile)
setAgentDetails();
string uri = "/test-uri";
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto upload_res = messaging_comp.uploadFile(uri, path, category, message_metadata);
auto upload_res = messaging_comp.uploadFile(uri, path, category, conn_metadata);
ASSERT_TRUE(upload_res.ok());
EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
}
@@ -185,16 +194,18 @@ TEST_F(TestMessagingComp, testDownloadFile)
string uri = "/test-uri";
HTTPMethod method = HTTPMethod::GET;
MessageCategory category = MessageCategory::GENERIC;
MessageMetadata message_metadata;
MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
Connection conn(conn_key, message_metadata);
Flags<MessageConnectionConfig> conn_flags;
conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
Connection conn(conn_key, conn_metadata);
EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
.WillOnce(Return(conn));
HTTPResponse res(HTTPStatusCode::HTTP_OK, "");
EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, message_metadata);
auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, conn_metadata);
ASSERT_TRUE(upload_res.ok());
EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK);
}

8
core/metric/generic_metric.cc
View File

@@ -65,7 +65,8 @@ GenericMetric::init(
const ReportIS::IssuingEngine &_issuing_engine,
chrono::seconds _report_interval,
bool _reset,
Audience _audience
Audience _audience,
bool _force_buffering
)
{
i_mainloop = Singleton::Consume<I_MainLoop>::by<GenericMetric>();
@@ -76,6 +77,7 @@ GenericMetric::init(
team = _team;
issuing_engine = _issuing_engine;
audience = _audience;
force_buffering = _force_buffering;
i_mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
@@ -233,7 +235,9 @@ GenericMetric::sendLog(const LogRest &metric_client_rest) const
HTTPMethod::POST,
fog_metric_uri,
metric_client_rest,
MessageCategory::METRIC
MessageCategory::METRIC,
MessageMetadata(),
force_buffering
);
}

11
core/metric/metric_ut/metric_ut.cc
View File

@@ -215,6 +215,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -283,6 +284,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -353,6 +355,7 @@ TEST_F(MetricTest, basicMetricTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -433,6 +436,7 @@ TEST_F(MetricTest, printMetricsTest)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).Times(AnyNumber());
@@ -504,6 +508,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -571,6 +576,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -638,6 +644,7 @@ TEST_F(MetricTest, metricTestWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -727,6 +734,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -801,6 +809,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -869,6 +878,7 @@ TEST_F(MetricTest, generateReportWithReset)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
@@ -1018,6 +1028,7 @@ TEST_F(MetricTest, testMapMetric)
"/api/v1/agents/events",
_,
MessageCategory::METRIC,
_,
_
)).WillRepeatedly(SaveArg<2>(&message_body));
routine();

4
core/rest/rest_server.cc
View File

@@ -133,7 +133,7 @@ RestServer::Impl::init()
auto is_primary = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>("Is Rest primary routine");
id = mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
I_MainLoop::RoutineType::RealTime,
fd,
[&] () { this->startNewConnection(); },
"REST server listener",
@@ -174,7 +174,7 @@ RestServer::Impl::startNewConnection() const
RestConn conn(new_socket, mainloop, this);
mainloop->addFileRoutine(
I_MainLoop::RoutineType::Offline,
I_MainLoop::RoutineType::RealTime,
new_socket,
[conn] () { conn.parseConn(); },
"REST server connection handler"