github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

2024 April 14th update

Browse Source
This commit is contained in:
Ned Wright
2024-04-14 12:55:54 +00:00
parent 7a7f65a77a
commit 942b2ef8b4
79 changed files with 1800 additions and 3778 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
core/config/config.cc
View File

@@ -17,11 +17,11 @@
#include <dirent.h>
#include <algorithm>
#include <fstream>
#include <boost/regex.hpp>
#include <iostream>
#include <cctype>
#include "agent_core_utilities.h"
#include "cereal/archives/json.hpp"
#include "debug.h"
#include "cereal/external/rapidjson/error/en.h"
#include "include/profile_settings.h"
@@ -158,6 +158,7 @@ private:
vector<string> fillMultiTenantConfigFiles(const map<string, set<string>> &tenants);
vector<string> fillMultiTenantExpectedConfigFiles(const map<string, set<string>> &tenants);
map<string, string> getProfileAgentSetting() const;
void resolveVsId() const;
string
getActiveTenant() const
@@ -243,7 +244,7 @@ private:
dbgTrace(D_CONFIG) << "File system path reloaded: " << config_directory_path;
}
bool
void
sendOrchestatorReloadStatusMsg(const LoadNewConfigurationStatus &status)
{
I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<ConfigComponent>();
@@ -261,7 +262,7 @@ private:
MessageMetadata secondary_port_req_md("127.0.0.1", 7778);
secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
service_config_status = messaging->sendSyncMessageWithoutResponse(
messaging->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
"/set-reconf-status",
status,
@@ -269,11 +270,6 @@ private:
secondary_port_req_md
);
}
if (!service_config_status) {
dbgWarning(D_CONFIG) << "Unsuccessful attempt to send configuration reload status";
return false;
}
return true;
}
unordered_map<TenantProfilePair, map<vector<string>, PerContextValue>> configuration_nodes;
@@ -339,6 +335,9 @@ void
ConfigComponent::Impl::init()
{
reloadFileSystemPaths();
resolveVsId();
tenant_manager = Singleton::Consume<I_TenantManager>::by<ConfigComponent>();
if (!Singleton::exists<I_MainLoop>()) return;
@@ -354,8 +353,7 @@ ConfigComponent::Impl::init()
}
}
static
bool
static bool
checkContext(const shared_ptr<EnvironmentEvaluator<bool>> &ctx)
{
if (ctx == nullptr) return true;
@@ -937,19 +935,31 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version
mainloop->stop(routine_id);
LoadNewConfigurationStatus finished(id, service_name, !res, true);
if (!res) finished.setError("Failed to reload configuration");
I_TimeGet *time = Singleton::Consume<I_TimeGet>::by<ConfigComponent>();
auto send_status_time_out = time->getMonotonicTime() + chrono::seconds(180);
while (time->getMonotonicTime() < send_status_time_out) {
if (sendOrchestatorReloadStatusMsg(finished)) break;
mainloop->yield(chrono::seconds(1));
}
if (time->getMonotonicTime() >= send_status_time_out) {
dbgWarning(D_CONFIG) << "Failed to send configuration reload status(finish) to the orchestrator";
}
sendOrchestatorReloadStatusMsg(finished);
is_continuous_report = false;
}
void
ConfigComponent::Impl::resolveVsId() const
{
const string &path = getConfigurationFlag("filesystem_path");
size_t vs_pos = path.rfind("/vs");
if (vs_pos == string::npos) return;
string vs_id = path.substr(vs_pos + 3);
if (!vs_id.empty() && all_of(vs_id.begin(), vs_id.end(), ::isdigit) && vs_id.size() < 6) {
dbgDebug(D_CONFIG) << "Identified VSX installation, VS ID: " << vs_id;
Singleton::Consume<I_Environment>::by<ConfigComponent>()->registerValue("VS ID", vs_id);
return;
}
dbgWarning(D_CONFIG) << "Possible VSX installation but VS ID is invalid, VS ID: " << vs_id;
return;
}
ConfigComponent::ConfigComponent() : Component("ConfigComponent"), pimpl(make_unique<Impl>()) {}
ConfigComponent::~ConfigComponent() {}

9
core/include/services_sdk/interfaces/messaging/messaging_enums.h
View File

@@ -80,4 +80,13 @@ enum class HTTPStatusCode
HTTP_SUSPEND = -2
};
enum class BioConnectionStatus
{
SUCCESS,
SHOULD_RETRY,
SHOULD_NOT_RETRY,
COUNT
};
#endif // __MESSAGING_ENUMS_H__

1
core/include/services_sdk/resources/debug_flags.h
View File

@@ -134,6 +134,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_AGENT_DETAILS, D_ORCHESTRATOR)
DEFINE_FLAG(D_LOCAL_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_NGINX_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_SERVICE_CONTROLLER, D_ORCHESTRATOR)
DEFINE_FLAG(D_GRADUAL_DEPLOYMENT, D_COMPONENT)
DEFINE_FLAG(D_SDWAN, D_COMPONENT)

43
core/include/services_sdk/resources/intelligence_invalidation.h
View File

@@ -18,6 +18,7 @@
#include <map>
#include <string>
#include <set>
#include <cereal/archives/json.hpp>
#include "maybe_res.h"
#include "enum_array.h"
@@ -31,26 +32,46 @@ enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED };
enum class InvalidationType { ADD, DELETE, UPDATE };
class StrAttributes
{
public:
StrAttributes() = default;
StrAttributes & addStringAttr(const std::string &attr, const std::string &val);
StrAttributes & addStringSetAttr(const std::string &attr, const std::set<std::string> &val);
Maybe<std::string, void> getStringAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
Maybe<std::string, void> genObject() const;
bool isEmpty() const;
bool matches(const StrAttributes &other) const;
void serialize(cereal::JSONInputArchive &ar);
void performOutputingSchema(std::ostream &, int);
private:
bool hasAttr(const std::string &key, const std::string &value) const;
std::map<std::string, std::string> string_attr;
std::map<std::string, std::set<std::string>> set_string_attr;
};
class Invalidation
{
public:
Invalidation(const std::string &class_value);
Invalidation & setClassifier(ClassifierType type, const std::string &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val, bool is_main = true);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val, bool is_main = true);
Invalidation & addMainAttr(const StrAttributes &attr);
Invalidation & addAttr(const StrAttributes &attr);
Invalidation & setSourceId(const std::string &id);
Invalidation & setObjectType(ObjectType type);
Invalidation & setInvalidationType(InvalidationType type);
std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
Maybe<std::string, void> getStringMainAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetMainAttr(const std::string &attr) const;
Maybe<std::string, void> getStringAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
std::vector<StrAttributes> getMainAttributes() const { return main_attributes; }
std::vector<StrAttributes> getAttributes() const { return attributes; }
const Maybe<std::string, void> & getSourceId() const { return source_id; }
const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
InvalidationType getInvalidationType() const { return invalidation_type; }
Maybe<std::string, void> getRegistrationID() const;
bool report(I_Intelligence_IS_V2 *interface) const;
@@ -64,18 +85,16 @@ public:
bool matches(const Invalidation &other) const;
private:
bool hasMainAttr(const std::string &key, const std::string &value) const;
bool hasAttr(const std::string &key, const std::string &value) const;
bool attr_matches(const std::vector<StrAttributes> &current, const std::vector<StrAttributes> &other) const;
EnumArray<ClassifierType, std::string, 6> classifiers;
std::map<std::string, std::string> string_main_attr;
std::map<std::string, std::set<std::string>> set_string_main_attr;
std::map<std::string, std::string> string_attr;
std::map<std::string, std::set<std::string>> set_string_attr;
std::vector<StrAttributes> main_attributes;
std::vector<StrAttributes> attributes;
Maybe<std::string, void> source_id;
Maybe<ObjectType, void> object_type;
InvalidationType invalidation_type = InvalidationType::ADD;
Maybe<uint, void> listening_id;
Maybe<std::string, void> registration_id;
};
} // namespace Intelligence

2
core/intelligence_is_v2/include/intelligence_request.h
View File

@@ -34,9 +34,7 @@ public:
bool isPagingActivated() const;
Maybe<bool> isPagingFinished() const;
Maybe<Intelligence_IS_V2::CursorState> getPagingStatus() const;
bool loadJson(const std::string &json);
Maybe<std::string> genJson() const;
Maybe<std::string> getResponseFromFog() const;
size_t getSize() const { return queries.size(); }
bool isBulk() const { return is_bulk; }

2
core/intelligence_is_v2/include/intelligence_server.h
View File

@@ -33,7 +33,7 @@ private:
Maybe<Response> sendQueryObjectToLocalServer(bool is_primary_port);
Maybe<Response> sendQueryMessage();
Maybe<Response> sendMessage();
Maybe<Response> createResponse();
Maybe<Response> createResponse(const std::string &response_body);
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;

81
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -37,14 +37,12 @@ static const string registration_uri = "/api/v2/intelligence/invalidation/regist
class I_InvalidationCallBack
{
public:
virtual void performCallBacks(const Invalidation &invalidation) const = 0;
virtual void performCallBacks(const Invalidation &invalidation, const string &registration_id) const = 0;
protected:
virtual ~I_InvalidationCallBack() {}
};
using MainAttrTypes = SerializableMultiMap<string, set<string>>;
static const map<string, Intelligence::ObjectType> object_names = {
{ "asset", Intelligence::ObjectType::ASSET },
{ "zone", Intelligence::ObjectType::ZONE },
@@ -54,6 +52,12 @@ static const map<string, Intelligence::ObjectType> object_names = {
{ "shortLived", Intelligence::ObjectType::SHORTLIVED }
};
static const map<string, Intelligence::InvalidationType> invalidation_type_names = {
{ "add", Intelligence::InvalidationType::ADD },
{ "delete", Intelligence::InvalidationType::DELETE },
{ "update", Intelligence::InvalidationType::UPDATE }
};
class InvalidationRegistration
{
public:
@@ -108,11 +112,22 @@ public:
do {
++running_id;
} while (callbacks.find(running_id) != callbacks.end());
auto invalidation_reg_id = invalidation.getRegistrationID();
if (invalidation_reg_id.ok()) registration_id_to_cb[*invalidation_reg_id] = cb;
callbacks.emplace(running_id, make_pair(invalidation, cb));
return running_id;
}
void erase(uint id) { callbacks.erase(id); }
void
erase(uint id)
{
auto actual_invalidation = callbacks.find(id);
if (actual_invalidation == callbacks.end()) return;
auto invalidation_reg_id = actual_invalidation->second.first.getRegistrationID();
if (invalidation_reg_id.ok()) registration_id_to_cb.erase(*invalidation_reg_id);
callbacks.erase(id);
}
bool empty() const { return callbacks.empty(); }
InvalidationRegistration::RestCall
@@ -128,9 +143,13 @@ public:
}
void
performCallBacks(const Invalidation &invalidation) const override
performCallBacks(const Invalidation &invalidation, const string &registration_id) const override
{
dbgDebug(D_INTELLIGENCE) << "Looking for callbacks for invalidation " << invalidation.genObject();
if (registration_id != "") {
auto invalidation_cb = registration_id_to_cb.find(registration_id);
if (invalidation_cb != registration_id_to_cb.end()) return invalidation_cb->second(invalidation);
}
for (auto &registed_invalidation : callbacks) {
dbgTrace(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject();
performCallBacksImpl(invalidation, registed_invalidation.second);
@@ -140,20 +159,22 @@ public:
private:
void
performCallBacksImpl(
const Invalidation &actual_invalidation,
const pair<Invalidation, function<void(const Invalidation &)>> &invalidation_and_cb
const Invalidation &actual_invalidation,
const pair<Invalidation, function<void(const Invalidation &)>> &invalidation_and_cb
) const
{
auto &registereed_invalidation = invalidation_and_cb.first;
auto &cb = invalidation_and_cb.second;
if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation);
if (!registereed_invalidation.matches(actual_invalidation)) return;
cb(actual_invalidation);
}
map<uint, pair<Invalidation, function<void(const Invalidation &)>>> callbacks;
map<string, function<void(const Invalidation &)>> registration_id_to_cb;
uint running_id = 0;
};
class RecieveInvalidation : public ServerRest
class ReceiveInvalidation : public ServerRest
{
public:
void
@@ -168,14 +189,14 @@ public:
if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get());
if (mainAttributes.isActive()) {
auto strings = getMainAttr<string>();
for (const auto &value : strings) {
invalidation.setStringAttr(value.first, value.second);
for (auto &vec_entry : mainAttributes.get()) {
invalidation.addMainAttr(vec_entry);
}
}
auto string_sets = getMainAttr<set<string>>();
for (const auto &value : string_sets) {
invalidation.setStringSetAttr(value.first, value.second);
if (attributes.isActive()) {
for (auto &vec_entry : attributes.get()) {
invalidation.addAttr(vec_entry);
}
}
@@ -186,25 +207,14 @@ public:
if (sourceId.isActive()) invalidation.setSourceId(sourceId.get());
string registration_id = "";
if (invalidationRegistrationId.isActive()) registration_id = invalidationRegistrationId.get();
auto i_cb = Singleton::Consume<I_InvalidationCallBack>::from<InvalidationCallBack>();
i_cb->performCallBacks(invalidation);
i_cb->performCallBacks(invalidation, registration_id);
}
private:
template <typename ValType>
map<string, ValType>
getMainAttr()
{
map<string, ValType> res;
for (auto &vec_entry : mainAttributes.get()) {
for (auto &attr : vec_entry.getMap<ValType>()) {
res[attr.first] = attr.second;
}
}
return res;
}
C2S_LABEL_PARAM(string, class_name, "class");
C2S_OPTIONAL_PARAM(string, category);
@@ -214,7 +224,9 @@ private:
C2S_OPTIONAL_PARAM(string, kind);
C2S_OPTIONAL_PARAM(string, objectType);
C2S_OPTIONAL_PARAM(string, sourceId);
C2S_OPTIONAL_PARAM(vector<MainAttrTypes>, mainAttributes);
C2S_OPTIONAL_PARAM(string, invalidationRegistrationId);
C2S_OPTIONAL_PARAM(vector<StrAttributes>, mainAttributes);
C2S_OPTIONAL_PARAM(vector<StrAttributes>, attributes);
};
class IntelligenceComponentV2::Impl
@@ -238,7 +250,7 @@ public:
);
auto rest_api = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
rest_api->addRestCall<RecieveInvalidation>(RestAction::SET, "new-invalidation/source/invalidation");
rest_api->addRestCall<ReceiveInvalidation>(RestAction::SET, "new-invalidation/source/invalidation");
}
bool
@@ -256,8 +268,9 @@ public:
registerInvalidation(const Invalidation &invalidation, const function<void(const Invalidation &)> &cb) override
{
if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation");
if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation");
return invalidations.emplace(invalidation, cb);
auto res = invalidations.emplace(invalidation, cb);
sendReccurringInvalidationRegistration();
return res;
}
void

550
core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
View File

@@ -15,6 +15,98 @@ using namespace testing;
static const string invalidation_uri = "/api/v2/intelligence/invalidation";
TEST(StringAttributesBasic, SettersAndGetters)
{
StrAttributes string_attributes;
EXPECT_TRUE(string_attributes.isEmpty());
EXPECT_FALSE(string_attributes.getStringAttr("attr1").ok());
EXPECT_FALSE(string_attributes.getStringSetAttr("attr2").ok());
set<string> vals = { "2", "3" };
string_attributes
.addStringAttr("attr1", "1")
.addStringSetAttr("attr2", vals);
EXPECT_FALSE(string_attributes.isEmpty());
EXPECT_EQ(string_attributes.getStringAttr("attr1").unpack(), "1");
EXPECT_EQ(string_attributes.getStringSetAttr("attr2").unpack(), vals);
}
TEST(StringAttributesBasic, attr_schema)
{
set<string> vals = { "2", "3" };
auto string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringSetAttr("attr2", vals);
stringstream ss;
string_attributes.performOutputingSchema(ss, 0);
string expected_schema =
"{\n"
" \"attr1\": \"1\",\n"
" \"attr2\": [\n"
" \"2\",\n"
" \"3\"\n"
" ]\n"
"}";
EXPECT_EQ(ss.str(), expected_schema);
}
TEST(StringAttributesBasic, Matching)
{
set<string> vals = { "2", "3" };
auto base_string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringAttr("attr2", "2")
.addStringAttr("attr3", "3")
.addStringSetAttr("attr4", vals);
auto matching_string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringAttr("attr2", "2")
.addStringAttr("attr3", "3")
.addStringSetAttr("attr4", vals)
.addStringAttr("attr5", "6")
.addStringSetAttr("attr6", vals);
EXPECT_TRUE(base_string_attributes.matches(matching_string_attributes));
auto not_matching_string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringAttr("attr2", "2")
.addStringSetAttr("attr4", vals)
.addStringAttr("attr3", "6");
EXPECT_FALSE(base_string_attributes.matches(not_matching_string_attributes));
auto missing_attr_string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringSetAttr("attr2", vals);
EXPECT_FALSE(base_string_attributes.matches(missing_attr_string_attributes));
set<string> vals2 = { "1", "5", "2", "3" };
auto has_extra_value_string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringAttr("attr2", "2")
.addStringAttr("attr3", "3")
.addStringSetAttr("attr4", vals2);
EXPECT_TRUE(base_string_attributes.matches(has_extra_value_string_attributes));
}
TEST(StringAttributesBasic, genObject)
{
set<string> vals = { "2", "3" };
auto string_attributes = StrAttributes()
.addStringAttr("attr1", "1")
.addStringSetAttr("attr2", vals);
string expected_json = "{ \"attr1\": \"1\", \"attr2\": [ \"2\", \"3\" ] }";
EXPECT_EQ(string_attributes.genObject().unpack(), expected_json);
}
TEST(InvalidationBasic, SettersAndGetters)
{
Invalidation invalidation("aaa");
@@ -27,33 +119,37 @@ TEST(InvalidationBasic, SettersAndGetters)
EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), "");
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD);
EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok());
EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok());
EXPECT_FALSE(invalidation.getStringAttr("attr1").ok());
EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok());
EXPECT_TRUE(invalidation.getMainAttributes().empty());
EXPECT_TRUE(invalidation.getAttributes().empty());
EXPECT_FALSE(invalidation.getSourceId().ok());
EXPECT_FALSE(invalidation.getObjectType().ok());
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
auto main_attr = StrAttributes()
.addStringAttr("main_attr1", "1")
.addStringSetAttr("main_attr2", main_vals);
auto attr = StrAttributes()
.addStringAttr("attr1", "4")
.addStringSetAttr("attr2", vals);
invalidation
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.addMainAttr(main_attr)
.addAttr(attr)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb");
EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc");
EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1");
EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals);
EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4");
EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(invalidation.getMainAttributes().begin()->getStringAttr("main_attr1").unpack(), "1");
EXPECT_EQ(invalidation.getMainAttributes().begin()->getStringSetAttr("main_attr2").unpack(), main_vals);
EXPECT_EQ(invalidation.getAttributes().begin()->getStringAttr("attr1").unpack(), "4");
EXPECT_EQ(invalidation.getAttributes().begin()->getStringSetAttr("attr2").unpack(), vals);
EXPECT_EQ(invalidation.getSourceId().unpack(), "id");
EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET);
EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE);
@@ -63,84 +159,87 @@ TEST(InvalidationBasic, Matching)
{
set<string> main_vals = { "2", "3" };
set<string> vals = { "5", "6" };
auto main_attr = StrAttributes()
.addStringAttr("main_attr1", "1")
.addStringSetAttr("main_attr2", main_vals);
auto attr = StrAttributes()
.addStringAttr("attr1", "4")
.addStringSetAttr("attr2", vals);
auto base_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false);
.addMainAttr(main_attr)
.addAttr(attr);
auto matching_main_attr = StrAttributes()
.addStringAttr("main_attr1", "1")
.addStringSetAttr("main_attr2", main_vals)
.addStringAttr("main_attr3", "6");
auto matching_attr = StrAttributes()
.addStringAttr("attr1", "4")
.addStringSetAttr("attr2", vals)
.addStringAttr("attr3", "7");
auto matching_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr3", "7", false)
.addMainAttr(matching_main_attr)
.addAttr(matching_attr)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::ADD);
EXPECT_TRUE(base_invalidation.matches(matching_invalidation));
auto not_matching_invalidation_type = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET)
.setInvalidationType(InvalidationType::DELETE);
EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type));
auto missing_attr_main = StrAttributes()
.addStringAttr("main_attr1", "1")
.addStringAttr("main_attr2", "2")
.addStringAttr("main_attr3", "6");
auto missing_attr_invalidation_main = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringAttr("main_attr2", "2")
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.addMainAttr(missing_attr_main)
.addAttr(matching_attr)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main));
auto missing_attr = StrAttributes()
.addStringAttr("attr1", "4")
.addStringAttr("attr2", "2")
.addStringAttr("attr3", "7");
auto missing_attr_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringAttr("main_attr1", "1")
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringAttr("attr2", "2", false)
.setStringAttr("attr3", "7", false)
.addMainAttr(matching_main_attr)
.addAttr(missing_attr)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation));
set<string> vals2 = { "1", "5" };
auto extra_value_main_attr = StrAttributes()
.addStringSetAttr("main_attr1", vals2)
.addStringSetAttr("main_attr2", main_vals)
.addStringAttr("main_attr3", "6");
auto has_extra_value_invalidation = Invalidation("aaa")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::GROUP, "ddd")
.setStringSetAttr("main_attr1", vals2)
.setStringSetAttr("main_attr2", main_vals)
.setStringAttr("main_attr3", "6")
.setStringAttr("attr1", "4", false)
.setStringSetAttr("attr2", vals, false)
.setStringAttr("attr3", "7", false)
.addMainAttr(extra_value_main_attr)
.addAttr(matching_attr)
.setSourceId("id")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -180,6 +279,8 @@ public:
conf.preload();
intelligence.preload();
intelligence.init();
main_attr.addStringAttr("attr2", "2");
attr.addStringAttr("attr3", "3");
}
bool
@@ -189,6 +290,8 @@ public:
return true;
}
StrAttributes main_attr;
StrAttributes attr;
StrictMock<MockMessaging> messaging_mock;
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> mock_time;
@@ -208,7 +311,7 @@ public:
TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setObjectType(Intelligence::ObjectType::ASSET);
@@ -219,8 +322,8 @@ TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation)
TEST_F(IntelligenceInvalidation, sending_public_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.addMainAttr(main_attr)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -254,11 +357,51 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, multiple_assets_invalidation)
{
auto main_attr_2 = StrAttributes()
.addStringAttr("attr2", "22")
.addStringSetAttr("attr3", {"33", "44"});
auto invalidation = Invalidation("aaa")
.addMainAttr(main_attr)
.addMainAttr(main_attr_2)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string invalidation_json;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(
SaveArg<2>(&invalidation_json),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
EXPECT_TRUE(invalidation.report(i_intelligence));
string expected_json =
"{ \"invalidations\": [ { "
"\"class\": \"aaa\", "
"\"category\": \"bbb\", "
"\"family\": \"ccc\", "
"\"objectType\": \"asset\", "
"\"invalidationType\": \"add\", "
"\"sourceId\": \"id\", "
"\"mainAttributes\": [ { \"attr2\": \"2\" }, { \"attr2\": \"22\", \"attr3\": [ \"33\", \"44\" ] } ], "
"\"attributes\": [ { \"attr3\": \"3\" } ]"
" } ] }";
EXPECT_EQ(invalidation_json, expected_json);
}
TEST_F(IntelligenceInvalidation, sending_private_invalidation)
{
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringAttr("attr3", "3", false)
.addMainAttr(main_attr)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -319,10 +462,10 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
set<string> vals = { "11", "55", "22" };
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.setStringSetAttr("attr3", vals, false)
.addMainAttr(main_attr)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -344,10 +487,89 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\""));
EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\""));
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]"));
EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": \"3\" } ]"));
EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
}
TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto multiple_assets_main_attr1 = StrAttributes()
.addStringAttr("attr2", "22");
auto multiple_assets_main_attr2 = StrAttributes()
.addStringAttr("attr2", "222");
auto multiple_assets_main_attr3 = StrAttributes()
.addStringAttr("attr2", "2222")
.addStringSetAttr("attr3", {"3333", "4444"});
auto invalidation = Invalidation("aaa")
.addMainAttr(multiple_assets_main_attr1)
.addMainAttr(multiple_assets_main_attr2)
.addMainAttr(multiple_assets_main_attr3)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
string body;
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillOnce(DoAll(
SaveArg<2>(&body),
Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
));
EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
EXPECT_THAT(
body,
HasSubstr(
"\"mainAttributes\": [ "
"{ \"attr2\": \"22\" }, "
"{ \"attr2\": \"222\" }, "
"{ \"attr2\": \"2222\", \"attr3\": [ \"3333\", \"4444\" ] } "
"]"
)
);
}
TEST_F(IntelligenceInvalidation, register_incomplit_invalidation)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto invalidation = Invalidation("aaa")
.addMainAttr(main_attr)
.addAttr(attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_FALSE(i_intelligence->registerInvalidation(invalidation, callback).ok());
}
TEST_F(IntelligenceInvalidation, invalidation_callback)
{
stringstream configuration;
@@ -363,7 +585,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -377,8 +599,10 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
set<string> vals = { "1", "5", "2" };
auto test_main_attr = StrAttributes()
.addStringSetAttr("attr2", vals);
auto invalidation2 = Invalidation("aaa")
.setStringSetAttr("attr2", vals)
.addMainAttr(test_main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -389,7 +613,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1);
EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals);
EXPECT_EQ(recieved_invalidations[0].getMainAttributes().begin()->getStringSetAttr("attr2").unpack(), vals);
}
TEST_F(IntelligenceInvalidation, delete_invalidation_callback)
@@ -407,7 +631,7 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -421,9 +645,8 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback)
auto callback_id = i_intelligence->registerInvalidation(invalidation, callback);
i_intelligence->unregisterInvalidation(*callback_id);
set<string> vals = { "1", "5", "2" };
auto invalidation2 = Invalidation("aaa")
.setStringSetAttr("attr2", vals)
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -451,7 +674,7 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling)
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -466,9 +689,8 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling)
invalidation.stopListening(i_intelligence);
set<string> vals = { "1", "5", "2" };
auto invalidation2 = Invalidation("aaa")
.setStringSetAttr("attr2", vals)
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -498,7 +720,7 @@ TEST_F(IntelligenceInvalidation, routine_registration)
routine();
auto invalidation = Invalidation("aaa")
.setStringAttr("attr2", "2")
.addMainAttr(main_attr)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
@@ -526,3 +748,191 @@ TEST_F(IntelligenceInvalidation, routine_registration)
EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\""));
EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
}
TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto base_main_attr2 = StrAttributes()
.addStringAttr("attr3", "3");
auto invalidation_to_register = Invalidation("aaa")
.addMainAttr(main_attr)
.addMainAttr(base_main_attr2)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
invalidation_to_register.startListening(i_intelligence, callback);
auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); });
auto not_matching_main_attributes = StrAttributes()
.addStringAttr("attr3", "4");
auto not_matching_invalidation = Invalidation("aaa")
.addMainAttr(not_matching_main_attributes)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json1;
json1 << not_matching_invalidation.genObject();
mock_invalidation->performRestCall(json1);
EXPECT_EQ(recieved_invalidations.size(), 0);
auto matching_second_main_attribute = StrAttributes()
.addStringAttr("attr3", "3");
auto matching_invalidation = Invalidation("aaa")
.addMainAttr(matching_second_main_attribute)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
stringstream json2;
json2 << matching_invalidation.genObject();
mock_invalidation->performRestCall(json2);
EXPECT_EQ(recieved_invalidations.size(), 1);
}
TEST_F(IntelligenceInvalidation, invalidation_cb_match_2_registred_assets)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto base_main_attr2 = StrAttributes()
.addStringAttr("attr3", "3");
auto invalidation_to_register = Invalidation("aaa")
.addMainAttr(main_attr)
.addMainAttr(base_main_attr2)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
invalidation_to_register.startListening(i_intelligence, callback);
auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); });
auto matching_second_main_attribute = StrAttributes()
.addStringAttr("attr3", "3");
auto matching_invalidation = Invalidation("aaa")
.addMainAttr(matching_second_main_attribute)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
auto invalidation_2_to_register = Invalidation("aaa")
.addMainAttr(base_main_attr2)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
invalidation_2_to_register.startListening(i_intelligence, callback);
auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); });
stringstream json;
json << matching_invalidation.genObject();
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 2);
}
TEST_F(IntelligenceInvalidation, invalidation_cb_match_by_registration_id)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
auto base_main_attr2 = StrAttributes()
.addStringAttr("attr3", "3");
auto invalidation_to_register = Invalidation("aaa")
.addMainAttr(main_attr)
.addMainAttr(base_main_attr2)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
EXPECT_CALL(
messaging_mock,
sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
).Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
invalidation_to_register.startListening(i_intelligence, callback);
auto stop_listening = make_scope_exit([&] { invalidation_to_register.stopListening(i_intelligence); });
auto matching_second_main_attribute = StrAttributes()
.addStringAttr("attr3", "3");
auto matching_invalidation = Invalidation("aaa")
.addMainAttr(matching_second_main_attribute)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
auto invalidation_2_to_register = Invalidation("aaa")
.addMainAttr(base_main_attr2)
.setSourceId("id")
.setClassifier(ClassifierType::FAMILY, "ccc")
.setClassifier(ClassifierType::CATEGORY, "bbb")
.setObjectType(Intelligence::ObjectType::ASSET);
invalidation_2_to_register.startListening(i_intelligence, callback);
auto registration_id = invalidation_2_to_register.getRegistrationID();
auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); });
string modifiedJsonString = matching_invalidation.genObject().substr(2);
stringstream json;
json << "{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString;
cout << json.str() << endl;
mock_invalidation->performRestCall(json);
EXPECT_EQ(recieved_invalidations.size(), 1);
}

13
core/intelligence_is_v2/intelligence_request.cc
View File

@@ -113,16 +113,3 @@ IntelligenceRequest::genJson() const
return str_stream.str();
}
bool
IntelligenceRequest::loadJson(const string &json)
{
response_from_fog = json;
return true;
}
Maybe<string>
IntelligenceRequest::getResponseFromFog() const
{
return response_from_fog;
}

69
core/intelligence_is_v2/intelligence_server.cc
View File

@@ -140,64 +140,49 @@ Sender::sendQueryMessage()
Maybe<Response>
Sender::sendMessage()
{
if (server_ip.ok() || server_port.ok()) {
if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set");
if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set");
} else if (!server_ip.ok() && !server_port.ok()) {
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE
);
if (req_status.ok()) {
return createResponse();
};
auto response_error = req_status.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ " "
+ req_status.getErr().toString()
);
if (server_port.ok() && !server_ip.ok()) return genError("Can't send intelligence request. Server ip invalid");
if (server_ip.ok() && !server_port.ok()) return genError("Can't send intelligence request. Server port invalid");
auto req_md = server_ip.ok() ? MessageMetadata(*server_ip, *server_port, conn_flags) : MessageMetadata();
if (server_ip.ok()) {
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< *server_ip
<< " port: "
<< *server_port
<< " query_uri: "
<< (request.isBulk() ? queries_uri : query_uri);
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< *server_ip
<< " port: "
<< *server_port
<< " query_uri: "
<< (request.isBulk() ? queries_uri : query_uri);
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
auto json_body = request.genJson();
if (!json_body.ok()) return json_body.passErr();
auto req_data = i_message->sendSyncMessage(
HTTPMethod::POST,
request.isBulk() ? queries_uri : query_uri,
request,
*json_body,
MessageCategory::INTELLIGENCE,
req_md
);
if (!req_status.ok()) {
auto response_error = req_status.getErr().toString();
if (!req_data.ok()) {
auto response_error = req_data.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ req_data.getErr().getBody()
+ " "
+ req_status.getErr().toString()
+ req_data.getErr().toString()
);
};
return createResponse();
} else if (req_data->getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) {
return genError("Intelligence response is invalid. " + req_data->toString());
}
return createResponse(req_data->getBody());
}
Maybe<Response>
Sender::createResponse()
Sender::createResponse(const std::string &response_body)
{
auto mb_json_body = request.getResponseFromFog();
if (!mb_json_body.ok()) return mb_json_body.passErr();
Response response(*mb_json_body, request.getSize(), request.isBulk());
Response response(response_body, request.getSize(), request.isBulk());
auto load_status = response.load();
if (!load_status.ok()) return load_status.passErr();
return response;

277
core/intelligence_is_v2/invalidation.cc
View File

@@ -14,6 +14,9 @@
#include "intelligence_invalidation.h"
#include <sstream>
#include <boost/uuid/uuid_generators.hpp>
#include "boost/uuid/uuid_io.hpp"
#include "boost/uuid/uuid.hpp"
#include "i_intelligence_is_v2.h"
@@ -24,7 +27,8 @@ Invalidation::Invalidation(const string &class_value)
:
source_id(genError<void>()),
object_type(genError<void>()),
listening_id(genError<void>())
listening_id(genError<void>()),
registration_id(genError<void>())
{
setClassifier(ClassifierType::CLASS, class_value);
}
@@ -36,20 +40,6 @@ Invalidation::setClassifier(ClassifierType type, const string &val)
return *this;
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val, bool is_main)
{
is_main ? string_main_attr[attr] = val : string_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val, bool is_main)
{
is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setSourceId(const string &id)
{
@@ -71,38 +61,6 @@ Invalidation::setInvalidationType(InvalidationType type)
return *this;
}
Maybe<string, void>
Invalidation::getStringMainAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetMainAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_attr.find(attr);
if (val_ref == string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_attr.find(attr);
if (val_ref == set_string_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
@@ -113,6 +71,7 @@ Invalidation::report(I_Intelligence_IS_V2 *interface) const
Maybe<uint>
Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function<void(const Invalidation &)> &cb)
{
registration_id = to_string(boost::uuids::random_generator()());
auto res = interface->registerInvalidation(*this, cb);
if (res.ok()) listening_id = *res;
return res;
@@ -176,53 +135,29 @@ Invalidation::genObject() const
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (registration_id.ok()) invalidation <<", \"invalidationRegistrationId\": \"" << *registration_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
if (!main_attributes.empty()) {
invalidation << ", \"mainAttributes\": [ ";
bool first = true;
for (auto &attr : string_main_attr) {
for (auto &main_attr : main_attributes) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
auto val = main_attr.genObject();
if (!val.ok()) continue;
invalidation << *val;
first = false;
}
for (auto &attr : set_string_main_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
if (!string_attr.empty() || !set_string_attr.empty()) {
if (!attributes.empty()) {
invalidation << ", \"attributes\": [ ";
bool first = true;
for (auto &attr : string_attr) {
for (auto &attr : attributes) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << value << "\"";
internal_first = false;
}
invalidation << " ] }";
auto val = attr.genObject();
if (!val.ok()) continue;
invalidation << *val;
first = false;
}
@@ -237,7 +172,7 @@ Invalidation::genObject() const
bool
Invalidation::isLegalInvalidation() const
{
if (!set_string_main_attr.empty() || !string_main_attr.empty()) {
if (!main_attributes.empty() || !attributes.empty()) {
if (classifiers[ClassifierType::FAMILY] == "") return false;
}
@@ -253,6 +188,18 @@ Invalidation::isLegalInvalidation() const
template <>
class EnumCount<ClassifierType> : public EnumCountSpecialization<ClassifierType, 6> {};
bool
Invalidation::attr_matches(const vector<StrAttributes> &current, const vector<StrAttributes> &other) const
{
if (current.empty()) return true;
for (auto &attr : current) {
for(auto &other_attr : other) {
if (attr.matches(other_attr)) return true;
}
}
return false;
}
bool
Invalidation::matches(const Invalidation &other) const
{
@@ -264,53 +211,104 @@ Invalidation::matches(const Invalidation &other) const
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (invalidation_type != other.invalidation_type) return false;
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasMainAttr(key_value.first, key_value.second)) return false;
}
if (!attr_matches(main_attributes, other.getMainAttributes())) return false;
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasMainAttr(key_values.first, value)) return false;
}
}
for (auto &key_value : string_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
}
if(!attr_matches(attributes, other.getAttributes())) return false;
return true;
}
bool
Invalidation::hasMainAttr(const string &key, const string &value) const
Invalidation &
Invalidation::addAttr(const StrAttributes &attr)
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
attributes.emplace_back(attr);
return *this;
}
auto set_string_elem = set_string_main_attr.find(key);
if (set_string_elem != set_string_main_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
Invalidation &
Invalidation::addMainAttr(const StrAttributes &attr)
{
main_attributes.emplace_back(attr);
return *this;
}
Maybe<string, void>
Invalidation::getRegistrationID() const{
return registration_id;
}
StrAttributes &
StrAttributes::addStringAttr(const std::string &attr, const std::string &val)
{
string_attr[attr] = val;
return *this;
}
StrAttributes &
StrAttributes::addStringSetAttr(const std::string &attr, const std::set<std::string> &val)
{
set_string_attr[attr] = val;
return *this;
}
Maybe<std::string, void>
StrAttributes::getStringAttr(const std::string &attr) const
{
auto val_ref = string_attr.find(attr);
if (val_ref == string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<std::set<std::string>, void>
StrAttributes::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_attr.find(attr);
if (val_ref == set_string_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<std::string, void>
StrAttributes::genObject() const
{
stringstream attributes_ss;
if (string_attr.empty() && set_string_attr.empty()) return genError<void>();
bool first = true;
attributes_ss << "{ ";
for (auto &attr : string_attr) {
if (!first) attributes_ss << ", ";
attributes_ss << "\"" << attr.first << "\": \"" << attr.second << "\"";
first = false;
}
return false;
for (auto &attr : set_string_attr) {
if (!first) attributes_ss << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
attributes_ss << "\"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) attributes_ss << ", ";
attributes_ss << "\"" << value << "\"";
internal_first = false;
}
attributes_ss << " ]";
first = false;
}
attributes_ss << " }";
return attributes_ss.str();
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
StrAttributes::isEmpty() const
{
return string_attr.empty() && set_string_attr.empty();
}
bool
StrAttributes::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_attr.find(key);
if (string_elem != string_attr.end()) return string_elem->second == value;
@@ -322,3 +320,54 @@ Invalidation::hasAttr(const string &key, const string &value) const
return false;
}
bool
StrAttributes::matches(const StrAttributes &other) const
{
for (auto &key_value : string_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
}
return true;
}
void
StrAttributes::serialize(cereal::JSONInputArchive &ar)
{
SerializableMultiMap<string, set<string>> attributes_map;
attributes_map.load(ar);
string_attr = attributes_map.getMap<string>();
set_string_attr = attributes_map.getMap<set<string>>();
}
void
StrAttributes::performOutputingSchema(ostream &out, int level) {
bool first = true;
RestHelper::printIndent(out, level) << "{\n";
for (auto &attr : string_attr) {
if (!first) out << ",\n";
RestHelper::printIndent(out, level + 1) << "\"" << attr.first << "\": \"" << attr.second << "\"";
first = false;
}
for (auto &attr : set_string_attr) {
if (!first) out << ",\n";
RestHelper::printIndent(out, level + 1) << "\"" << attr.first << "\": [\n";
bool internal_first = true;
for (auto &value : attr.second) {
if (!internal_first) out << ",\n";
RestHelper::printIndent(out, level + 2) << "\"" << value << "\"";
internal_first = false;
}
out << "\n";
RestHelper::printIndent(out, level + 1) << "]\n";
first = false;
}
RestHelper::printIndent(out, level) << "}";
}

70
core/messaging/connection/connection.cc
View File

@@ -438,6 +438,26 @@ private:
}
// LCOV_EXCL_STOP
BioConnectionStatus
tryToBioConnect(const string &full_address)
{
BIO_set_conn_hostname(bio.get(), full_address.c_str());
BIO_set_nbio(bio.get(), 1);
auto bio_connect = BIO_do_connect(bio.get());
if (bio_connect > 0) return BioConnectionStatus::SUCCESS;
if (BIO_should_retry(bio.get())) return BioConnectionStatus::SHOULD_RETRY;
string bio_error = ERR_error_string(ERR_get_error(), nullptr);
dbgWarning(D_CONNECTION)
<< "Connection to: "
<< full_address
<< " failed and won't retry. Error: "
<< bio_error;
return BioConnectionStatus::SHOULD_NOT_RETRY;
}
Maybe<void>
connectToHost()
{
@@ -449,45 +469,43 @@ private:
}
dbgFlow(D_CONNECTION) << "Connecting to " << full_address;
BIO_set_conn_hostname(bio.get(), full_address.c_str());
BIO_set_nbio(bio.get(), 1);
I_MainLoop *i_mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
auto bio_connect = BIO_do_connect(bio.get());
BioConnectionStatus bio_connect = tryToBioConnect(full_address);
uint attempts_count = 0;
auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
while (i_time->getMonotonicTime() < conn_end_time && bio_connect <= 0) {
if (!BIO_should_retry(bio.get())) {
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
string bio_error = ERR_error_string(ERR_get_error(), nullptr);
return genError(
"Failed to connect to: " +
full_address +
", error: " +
bio_error +
". Connection suspended for 60 seconds");
}
attempts_count++;
if (!isBioSocketReady()) {
while (i_time->getMonotonicTime() < conn_end_time && bio_connect == BioConnectionStatus::SHOULD_RETRY) {
if (isBioSocketReady()) {
bio_connect = tryToBioConnect(full_address);
} else {
i_mainloop->yield((attempts_count % 10) == 0);
continue;
}
bio_connect = BIO_do_connect(bio.get());
}
if (bio_connect > 0) {
if (bio_connect == BioConnectionStatus::SUCCESS) {
if (isUnsecure() || isOverProxy()) return Maybe<void>();
return performHandshakeAndVerifyCert(i_time, i_mainloop);
}
if (bio_connect == BioConnectionStatus::SHOULD_NOT_RETRY) {
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
dbgWarning(D_CONNECTION)
<< "Connection to: "
<< full_address
<< " failed and will be suspended for 60 seconds";
return genError(full_address + ". There won't be a retry attempt.");
}
auto curr_time = chrono::duration_cast<chrono::seconds>(i_time->getMonotonicTime());
active = genError(curr_time + chrono::seconds(60));
return genError(
"Failed to establish new connection to: " +
full_address +
" after reaching timeout." +
" Connection suspended for 60 seconds");
dbgWarning(D_CONNECTION)
<< "Connection attempts to: "
<< full_address
<< " have reached timeout and will be suspended for 60 seconds";
return genError(full_address + ". Connection has reached timeout.");
}
Maybe<uint, HTTPResponse>
@@ -592,7 +610,7 @@ private:
auto send_size = sendData(request, data_left_to_send);
if (!send_size.ok()) return send_size.passErr();
data_left_to_send -= *send_size;
i_mainloop->yield(*send_size == 0); // We want to force waiting if we failed to send the data
i_mainloop->yield(*send_size == 0);
}
auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout();

2
core/messaging/messaging_comp/messaging_comp.cc
View File

@@ -62,7 +62,7 @@ MessagingComp::getConnection(MessageCategory category, const MessageMetadata &me
auto maybe_conn = i_conn->establishConnection(metadata, category);
if (!maybe_conn.ok()) {
dbgWarning(D_MESSAGING) << "Failed to establish connection: " << maybe_conn.getErr();
dbgWarning(D_MESSAGING) << maybe_conn.getErr();
}
return maybe_conn;
}

4
core/rest/rest_server.cc
View File

@@ -133,7 +133,7 @@ RestServer::Impl::init()
auto is_primary = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>("Is Rest primary routine");
id = mainloop->addFileRoutine(
I_MainLoop::RoutineType::RealTime,
I_MainLoop::RoutineType::Offline,
fd,
[&] () { this->startNewConnection(); },
"REST server listener",
@@ -174,7 +174,7 @@ RestServer::Impl::startNewConnection() const
RestConn conn(new_socket, mainloop, this);
mainloop->addFileRoutine(
I_MainLoop::RoutineType::RealTime,
I_MainLoop::RoutineType::Offline,
new_socket,
[conn] () { conn.parseConn(); },
"REST server connection handler"