Jul 5th update

This commit is contained in:
Ned Wright
2023-07-05 23:32:39 +00:00
parent 22f1a984aa
commit a59f079ef7
85 changed files with 2488 additions and 1754 deletions

View File

@@ -1,7 +1,7 @@
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc
)
add_subdirectory(intelligence_is_v2_ut)
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
)
add_subdirectory(intelligence_is_v2_ut)

View File

@@ -19,13 +19,200 @@
#include "config.h"
#include "table.h"
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_invalidation.h"
using namespace std;
using namespace chrono;
using namespace Intelligence_IS_V2;
using namespace Intelligence;
USE_DEBUG_FLAG(D_INTELLIGENCE);
static const string primary_port_setting = "local intelligence server primary port";
static const string secondary_port_setting = "local intelligence server secondary port";
static const string invalidation_uri = "/api/v2/intelligence/invalidation";
static const string registration_uri = "/api/v2/intelligence/invalidation/register";
class I_InvalidationCallBack
{
public:
virtual void performCallBacks(const Invalidation &invalidation) const = 0;
protected:
virtual ~I_InvalidationCallBack() {}
};
using MainAttrTypes = SerializableMultiMap<string, set<string>>;
static const map<string, Intelligence::ObjectType> object_names = {
{ "asset", Intelligence::ObjectType::ASSET },
{ "zone", Intelligence::ObjectType::ZONE },
{ "policyPackage", Intelligence::ObjectType::POLICY_PACKAGE },
{ "configuration", Intelligence::ObjectType::CONFIGURATION },
{ "session", Intelligence::ObjectType::SESSION }
};
class InvalidationRegistration
{
public:
class RestCall
{
public:
RestCall(const stringstream &input) : rest_body(input.str()) {}
Maybe<string> genJson() const { return rest_body; }
ostream & print(ostream &os) { return os << rest_body; }
private:
string rest_body;
};
void
addInvalidation(const Invalidation &invalidation)
{
if (!first) stream << ',';
stream << ' ' << invalidation.genObject();
first = false;
}
RestCall
genJson() const
{
stringstream res;
res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", ";
auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
res << "\"name\": \"" << details->getAgentId() << "\", ";
auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
res << "\"dataMap\": [";
res << stream.str();
res << " ] }";
return res;
}
private:
bool first = true;
stringstream stream;
};
class InvalidationCallBack : Singleton::Provide<I_InvalidationCallBack>::SelfInterface
{
public:
uint
emplace(const Invalidation &invalidation, function<void(const Invalidation &)> cb)
{
dbgDebug(D_INTELLIGENCE) << "registering " << invalidation.genObject();
do {
++running_id;
} while (callbacks.find(running_id) != callbacks.end());
callbacks.emplace(running_id, make_pair(invalidation, cb));
return running_id;
}
void erase(uint id) { callbacks.erase(id); }
bool empty() const { return callbacks.empty(); }
InvalidationRegistration::RestCall
getRegistration() const
{
InvalidationRegistration registration;
for (auto &registed_invalidation : callbacks) {
registration.addInvalidation(registed_invalidation.second.first);
}
return registration.genJson();
}
void
performCallBacks(const Invalidation &invalidation) const override
{
for (auto &registed_invalidation : callbacks) {
performCallBacksImpl(invalidation, registed_invalidation.second);
}
}
private:
void
performCallBacksImpl(
const Invalidation &actual_invalidation,
const pair<Invalidation, function<void(const Invalidation &)>> &invalidation_and_cb
) const
{
auto &registereed_invalidation = invalidation_and_cb.first;
auto &cb = invalidation_and_cb.second;
if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation);
}
map<uint, pair<Invalidation, function<void(const Invalidation &)>>> callbacks;
uint running_id = 0;
};
class RecieveInvalidation : public ServerRest
{
public:
void
doCall() override
{
Invalidation invalidation(class_name);
if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get());
if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get());
if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get());
if (order.isActive()) invalidation.setClassifier(ClassifierType::ORDER, order.get());
if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get());
if (mainAttributes.isActive()) {
auto strings = getMainAttr<string>();
for (const auto &value : strings) {
invalidation.setStringAttr(value.first, value.second);
}
auto string_sets = getMainAttr<set<string>>();
for (const auto &value : string_sets) {
invalidation.setStringSetAttr(value.first, value.second);
}
}
if (objectType.isActive()) {
auto type = object_names.find(objectType.get());
if (type != object_names.end()) invalidation.setObjectType(type->second);
}
if (sourceId.isActive()) invalidation.setSourceId(sourceId.get());
auto i_cb = Singleton::Consume<I_InvalidationCallBack>::from<InvalidationCallBack>();
i_cb->performCallBacks(invalidation);
}
private:
template <typename ValType>
map<string, ValType>
getMainAttr()
{
map<string, ValType> res;
for (auto &vec_entry : mainAttributes.get()) {
for (auto &attr : vec_entry.getMap<ValType>()) {
res[attr.first] = attr.second;
}
}
return res;
}
C2S_LABEL_PARAM(string, class_name, "class");
C2S_OPTIONAL_PARAM(string, category);
C2S_OPTIONAL_PARAM(string, family);
C2S_OPTIONAL_PARAM(string, group);
C2S_OPTIONAL_PARAM(string, order);
C2S_OPTIONAL_PARAM(string, kind);
C2S_OPTIONAL_PARAM(string, objectType);
C2S_OPTIONAL_PARAM(string, sourceId);
C2S_OPTIONAL_PARAM(vector<MainAttrTypes>, mainAttributes);
};
class IntelligenceComponentV2::Impl
:
Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
@@ -78,6 +265,37 @@ public:
message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
chrono::minutes(12),
[this] () { sendReccurringInvalidationRegistration(); },
"Sending intelligence invalidation"
);
auto rest_api = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
rest_api->addRestCall<RecieveInvalidation>(RestAction::SET, "new-invalidation/source/invalidation");
}
bool
sendInvalidation(const Invalidation &invalidation) const override
{
if (offline_mode_only) return false;
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
}
Maybe<uint>
registerInvalidation(const Invalidation &invalidation, const function<void(const Invalidation &)> &cb) override
{
if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation");
if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation");
return invalidations.emplace(invalidation, cb);
}
void
unregisterInvalidation(uint id) override
{
invalidations.erase(id);
}
I_Messaging *
@@ -102,11 +320,20 @@ public:
getOfflineInfoString(const SerializableQueryFilter &query) const override
{
string ip_attr_key = "mainAttributes.ip";
string identifier_value = move(query.getConditionValueByKey(ip_attr_key));
if (identifier_value == "") {
auto valueVariant = query.getConditionValueByKey(ip_attr_key);
if (!valueVariant.ok()) {
return genError("could not find IP main attribute in the given query.");
}
return offline_intelligence.getValueByIdentifier(identifier_value);
const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
if (const string* identifier_value = boost::get<string>(&value)) {
if (*identifier_value == "") {
return genError("Could not find IP main attribute in the given query.");
}
return offline_intelligence.getValueByIdentifier(*identifier_value);
}
return genError("Value is not of type string.");
}
bool
@@ -116,8 +343,157 @@ public:
}
private:
bool
hasLocalIntelligence() const
{
return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
}
bool
sendLocalInvalidation(const Invalidation &invalidation) const
{
dbgFlow(D_INTELLIGENCE) << "Starting local invalidation";
return sendLocalInvalidationImpl(invalidation) || sendGlobalInvalidation(invalidation);
}
bool
sendLocalInvalidationImpl(const Invalidation &invalidation) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
}
return
sendLocalInvalidationImpl(invalidation, *server, primary_port_setting) ||
sendLocalInvalidationImpl(invalidation, *server, secondary_port_setting);
}
bool
sendLocalInvalidationImpl(const Invalidation &invalidation, const string &server, const string &port_setting) const
{
dbgFlow(D_INTELLIGENCE) << "Sending to local intelligence";
auto port = getSetting<uint>("intelligence", port_setting);
if (!port.ok()) {
dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
return false;
}
dbgTrace(D_INTELLIGENCE)
<< "Invalidation value: "
<< (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
return message->sendNoReplyObject(
invalidation,
I_Messaging::Method::POST,
server,
*port,
Flags<MessageConnConfig>(),
invalidation_uri,
getHTTPHeaders(),
nullptr,
MessageTypeTag::INTELLIGENCE
);
}
bool
sendGlobalInvalidation(const Invalidation &invalidation) const
{
dbgFlow(D_INTELLIGENCE) << "Starting global invalidation";
dbgTrace(D_INTELLIGENCE)
<< "Invalidation value: "
<< (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
return message->sendNoReplyObject(
invalidation,
I_Messaging::Method::POST,
invalidation_uri,
getHTTPHeaders(),
nullptr,
true,
MessageTypeTag::INTELLIGENCE
);
}
string
getHTTPHeaders() const
{
auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
auto tenant = details->getTenantId();
if (tenant == "") tenant = "Global";
auto agent = details->getAgentId();
return "X-Tenant-Id: " + tenant + "\r\nX-Source-Id: " + agent;
}
bool
sendRegistration(const Invalidation &invalidation) const
{
if (offline_mode_only) return false;
InvalidationRegistration registration;
registration.addInvalidation(invalidation);
return sendLocalRegistrationImpl(registration.genJson());
}
bool
sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
}
return
sendLocalRegistrationImpl(registration, *server, primary_port_setting) ||
sendLocalRegistrationImpl(registration, *server, secondary_port_setting);
}
bool
sendLocalRegistrationImpl(
const InvalidationRegistration::RestCall &registration,
const string &server,
const string &port_setting
) const
{
dbgFlow(D_INTELLIGENCE) << "Sending to local registration";
auto port = getSetting<uint>("intelligence", port_setting);
if (!port.ok()) {
dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
return false;
}
dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
return message->sendNoReplyObject(
registration,
I_Messaging::Method::POST,
server,
*port,
Flags<MessageConnConfig>(),
registration_uri,
"",
nullptr,
MessageTypeTag::INTELLIGENCE
);
}
void
sendReccurringInvalidationRegistration() const
{
if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}
OfflineIntelligeceHandler offline_intelligence;
bool offline_mode_only = false;
InvalidationCallBack invalidations;
I_Messaging *message = nullptr;
I_TimeGet *timer = nullptr;
I_MainLoop *mainloop = nullptr;
@@ -141,8 +517,8 @@ IntelligenceComponentV2::preload()
registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
registerExpectedSetting<string>("intelligence", "local intelligence server ip");
registerExpectedSetting<uint>("intelligence", "local intelligence server secondary port");
registerExpectedSetting<uint>("intelligence", "local intelligence server primary port");
registerExpectedSetting<uint>("intelligence", primary_port_setting);
registerExpectedSetting<uint>("intelligence", secondary_port_setting);
registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy);
}

View File

@@ -1,7 +1,7 @@
file(COPY offline_intelligence_files_v2 DESTINATION .)
link_directories(${BOOST_ROOT}/lib)
add_unit_test(
intelligence_is_v2_ut
"query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc"
"intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")
file(COPY offline_intelligence_files_v2 DESTINATION .)
link_directories(${BOOST_ROOT}/lib)
add_unit_test(
intelligence_is_v2_ut
"query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc;invalidation_ut.cc"
"intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")

View File

@@ -9,6 +9,7 @@
#include "mock/mock_messaging.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_time_get.h"
#include "mock/mock_rest_api.h"
using namespace std;
using namespace testing;
@@ -35,8 +36,18 @@ public:
EXPECT_CALL(
mock_ml,
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _))
.WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _)
).WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
EXPECT_CALL(
mock_ml,
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(720000000), _, _, _)
).WillRepeatedly(Return(0));
EXPECT_CALL(
mock_rest,
mockRestCall(_, "new-invalidation/source/invalidation", _)
).WillRepeatedly(Return(true));
string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
@@ -52,6 +63,7 @@ public:
stringstream debug_output;
StrictMock<MockMainLoop> mock_ml;
StrictMock<MockRestApi> mock_rest;
NiceMock<MockTimeGet> mock_time;
::Environment env;
ConfigComponent conf;
@@ -853,3 +865,175 @@ TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
}
TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2)
{
string paging_in_progress_response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset1\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test1\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test1\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" },\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset2\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test2\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test2\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"inProgress\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"efgh\"\n"
"}\n"
);
string paging_done_response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset1\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test1\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test1\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" },\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset2\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test2\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test2\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"efgh\"\n"
"}\n"
);
EXPECT_CALL(
messaging_mock,
sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE))
.WillOnce(Return(paging_in_progress_response_str))
.WillOnce(Return(paging_done_response_str));
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true, AttributeKeyType::NONE);
request.activatePaging();
request.setAssetsLimit(10);
vector<AssetReply<Profile>> objects_reply;
vector<string> objects_ids;
do {
auto object_result = intell->queryIntelligence<Profile>(request, true);
if (!object_result.ok()) {
if (object_result.getErr() == "Query intelligence response with InProgress status") continue;
break;
}
objects_reply = object_result.unpack();
if (objects_reply.empty()) break;
for (const AssetReply<Profile> &current_object : objects_reply) {
if (current_object.getMainAttributes().empty()) {
continue;
}
const string &id = current_object.getMainAttributes().begin()->second[0];
objects_ids.push_back(id);
}
} while (!request.isPagingFinished());
EXPECT_EQ(objects_ids.size(), 2);
}

View File

@@ -10,29 +10,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(QueryRequestTestV2, QueryTest)
{
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
SerializableQueryFilter request_query1 = request.getQuery();
vector<SerializableQueryCondition> request_operands1 = request_query1.getConditionOperands();
SerializableQueryCondition request_condition = *request_operands1.begin();
EXPECT_EQ(request_query1.getOperator(), Operator::NONE);
EXPECT_EQ(request_condition.getKey(), "mainAttributes.phase");
EXPECT_EQ(request_condition.getValue(), "testing");
request.addCondition(Condition::EQUALS, "user1", "Omry");
request.addCondition(Condition::EQUALS, "user2", "Max");
SerializableQueryFilter request_query2 = request.getQuery();
vector<SerializableQueryCondition> request_operands2 = request_query2.getConditionOperands();
vector<SerializableQueryCondition>::iterator it = request_operands2.begin();
it++;
EXPECT_EQ(request_query2.getOperator(), Operator::AND);
EXPECT_EQ(it->getKey(), "mainAttributes.user1");
EXPECT_EQ(it->getValue(), "Omry");
it++;
EXPECT_EQ(it->getKey(), "mainAttributes.user2");
EXPECT_EQ(it->getValue(), "Max");
string output_json =
"{\n"
@@ -67,6 +46,38 @@ TEST(QueryRequestTestV2, QueryTest)
request.saveToJson(out_ar);
}
EXPECT_EQ(out.str(), output_json);
QueryRequest request2(Condition::GREATER_THAN, "prev_time", 1676887025952, true);
request2.addCondition(Condition::LESS_THAN, "curr_time", 1676887025958);
string output_json2=
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"and\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"greaterThan\",\n"
" \"key\": \"mainAttributes.prev_time\",\n"
" \"value\": 1676887025952\n"
" },\n"
" {\n"
" \"operator\": \"lessThan\",\n"
" \"key\": \"mainAttributes.curr_time\",\n"
" \"value\": 1676887025958\n"
" }\n"
" ]\n"
" }\n"
"}";
stringstream out2;
{
cereal::JSONOutputArchive out_ar2(out2);
request2.saveToJson(out_ar2);
}
EXPECT_EQ(out2.str(), output_json2);
}
TEST(QueryRequestTestV2, AttributesTest)
@@ -185,6 +196,104 @@ TEST(QueryRequestTestV2, OrQueryTest)
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, AndQueryTestThree)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
QueryRequest and_request_1_2 = request1 && (request2 && request3);
QueryRequest and_request_2_1 = (request1 && request2) && request3;
stringstream out_1_2;
{
cereal::JSONOutputArchive out_1_2_ar(out_1_2);
and_request_1_2.saveToJson(out_1_2_ar);
}
stringstream out_2_1;
{
cereal::JSONOutputArchive out_2_1_ar(out_2_1);
and_request_2_1.saveToJson(out_2_1_ar);
}
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"and\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing1\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing2\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing3\"\n"
" }\n"
" ]\n"
" }\n"
"}";
EXPECT_EQ(out_1_2.str(), output_json);
EXPECT_EQ(out_2_1.str(), output_json);
}
TEST(QueryRequestTestV2, OrQueryTestThree)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
QueryRequest or_request_1_2 = request1 || (request2 || request3);
QueryRequest or_request_2_1 = (request1 || request2) || request3;
stringstream out_1_2;
{
cereal::JSONOutputArchive out_1_2_ar(out_1_2);
or_request_1_2.saveToJson(out_1_2_ar);
}
stringstream out_2_1;
{
cereal::JSONOutputArchive out_2_1_ar(out_2_1);
or_request_2_1.saveToJson(out_2_1_ar);
}
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"or\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing1\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing2\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing3\"\n"
" }\n"
" ]\n"
" }\n"
"}";
EXPECT_EQ(out_1_2.str(), output_json);
EXPECT_EQ(out_2_1.str(), output_json);
}
TEST(QueryRequestTestV2, AndWithConditionQueryTest)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

View File

@@ -36,6 +36,8 @@ Intelligence_IS_V2::convertConditionTypeToString(const Condition &condition_type
{Condition::CONTAINS, "contains"},
{Condition::IN, "in"},
{Condition::NOT_IN, "notIn"},
{Condition::GREATER_THAN, "greaterThan"},
{Condition::LESS_THAN, "lessThan"},
};
auto condition_str = condition_type_to_string_map.find(condition_type);

View File

@@ -0,0 +1,240 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_invalidation.h"
#include <sstream>
#include "i_intelligence_is_v2.h"
using namespace Intelligence;
using namespace std;
Invalidation::Invalidation(const string &class_value)
:
source_id(genError<void>()),
object_type(genError<void>()),
listening_id(genError<void>())
{
setClassifier(ClassifierType::CLASS, class_value);
}
Invalidation &
Invalidation::setClassifier(ClassifierType type, const string &val)
{
classifiers[type] = val;
return *this;
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val)
{
string_main_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
{
set_string_main_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setSourceId(const string &id)
{
source_id = id;
return *this;
}
Invalidation &
Invalidation::setObjectType(ObjectType type)
{
object_type = type;
return *this;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
if (!isLegalInvalidation()) return false;
return interface->sendInvalidation(*this);
}
Maybe<uint>
Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function<void(const Invalidation &)> &cb)
{
auto res = interface->registerInvalidation(*this, cb);
if (res.ok()) listening_id = *res;
return res;
}
void
Invalidation::stopListening(I_Intelligence_IS_V2 *interface)
{
if (listening_id.ok()) interface->unregisterInvalidation(*listening_id);
}
static const map<Intelligence::ObjectType, string> convertObjectType = {
{ Intelligence::ObjectType::ASSET, "asset" },
{ Intelligence::ObjectType::ZONE, "zone" },
{ Intelligence::ObjectType::POLICY_PACKAGE, "policyPackage" },
{ Intelligence::ObjectType::CONFIGURATION, "configuration" },
{ Intelligence::ObjectType::SESSION, "session" }
};
Maybe<string>
Invalidation::genJson() const
{
if (!isLegalInvalidation()) return genError("Incomplete intelligence invalidation");
stringstream invalidation;
invalidation << "{ \"invalidations\": [ " << genObject() <<" ] }";
return invalidation.str();
}
string
Invalidation::genObject() const
{
stringstream invalidation;
invalidation << "{ \"class\": \"" << classifiers[ClassifierType::CLASS] << '"';
if (classifiers[ClassifierType::CATEGORY] != "") {
invalidation <<", \"category\": \"" << classifiers[ClassifierType::CATEGORY] << '"';
}
if (classifiers[ClassifierType::FAMILY] != "") {
invalidation <<", \"family\": \"" << classifiers[ClassifierType::FAMILY] << '"';
}
if (classifiers[ClassifierType::GROUP] != "") {
invalidation <<", \"group\": \"" << classifiers[ClassifierType::GROUP] << '"';
}
if (classifiers[ClassifierType::ORDER] != "") {
invalidation <<", \"order\": \"" << classifiers[ClassifierType::ORDER] << '"';
}
if (classifiers[ClassifierType::KIND] != "") {
invalidation <<", \"kind\": \"" << classifiers[ClassifierType::KIND] << '"';
}
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
invalidation << ", \"mainAttributes\": [ ";
bool first = true;
for (auto &attr : string_main_attr) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_main_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &val : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << val << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
invalidation << " }";
return invalidation.str();
}
bool
Invalidation::isLegalInvalidation() const
{
if (!set_string_main_attr.empty() || !string_main_attr.empty()) {
if (classifiers[ClassifierType::FAMILY] == "") return false;
}
bool is_prev_empty = false;
for (auto &classifer : classifiers) {
if (is_prev_empty && classifer != "") return false;
is_prev_empty = classifer == "";
}
return true;
}
template <>
class EnumCount<ClassifierType> : public EnumCountSpecialization<ClassifierType, 6> {};
bool
Invalidation::matches(const Invalidation &other) const
{
for (auto key : NGEN::Range<ClassifierType>()) {
if (classifiers[key] != "" && classifiers[key] != other.classifiers[key]) return false;
}
if (object_type.ok()) {
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
}
return true;
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
auto set_string_elem = set_string_main_attr.find(key);
if (set_string_elem != set_string_main_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
}
return false;
}

View File

@@ -0,0 +1,50 @@
// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/json_stream.h"
JsonStream::JsonStream(std::ostream *os, bool is_pretty) : std::ostream(this), os(os), is_pretty(is_pretty) {}
int
JsonStream::overflow(int c)
{
if (c != std::streambuf::traits_type::eof()) {
add(std::streambuf::traits_type::to_char_type(c));
}
return c;
}
void
JsonStream::emplace(char c)
{
*os << c;
}
void
JsonStream::add(char c)
{
if (is_pretty) {
emplace(c);
return;
}
if (is_prev_single_backslash) {
emplace(c);
is_prev_single_backslash = false;
return;
}
if (c == '"') in_string = !in_string;
if (c == '\\') is_prev_single_backslash = true;
if (in_string || !std::isspace(c)) emplace(c);
}

View File

@@ -23,19 +23,46 @@ using namespace Intelligence_IS_V2;
USE_DEBUG_FLAG(D_INTELLIGENCE);
struct Visitor : public boost::static_visitor<void>
{
Visitor(cereal::JSONOutputArchive& _ar) : ar(_ar) {}
template <typename T>
void operator()(const T& value)
{
ar(cereal::make_nvp("value", value));
}
private:
cereal::JSONOutputArchive &ar;
};
void
SerializableQueryCondition::save(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("operator", string(convertConditionTypeToString(condition_type))),
cereal::make_nvp("key", key),
cereal::make_nvp("value", value)
cereal::make_nvp("key", key)
);
Visitor visitor(ar);
boost::apply_visitor(visitor, value);
}
SerializableQueryFilter::SerializableQueryFilter(Condition condition_type, const string &key, const string &value)
{
condition_operands.push_back(SerializableQueryCondition(condition_type, key, value));
SerializableQueryFilter::SerializableQueryFilter(
Condition condition_type,
const std::string &key,
const std::string &value
) {
condition_operands.emplace_back(condition_type, key, value);
}
SerializableQueryFilter::SerializableQueryFilter(
Condition condition_type,
const std::string &key,
const int64_t &value
) {
condition_operands.emplace_back(condition_type, key, value);
}
void
@@ -52,31 +79,28 @@ void
SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const string &value)
{
if (queries_operands.size() > 0) {
SerializableQueryFilter new_query_filter(condition_type, key, value);
queries_operands.push_back(new_query_filter);
queries_operands.emplace_back(condition_type, key, value);
return;
}
if (condition_operands.size() == 1 && operator_type == Operator::NONE) {
operator_type = Operator::AND;
if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
condition_operands.emplace_back(condition_type, key, value);
}
void
SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const int64_t &value)
{
if (queries_operands.size() > 0) {
queries_operands.emplace_back(condition_type, key, value);
return;
}
SerializableQueryCondition cond(condition_type, key, value);
condition_operands.push_back(cond);
if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
condition_operands.emplace_back(condition_type, key, value);
}
void
SerializableQueryFilter::saveCondition(cereal::JSONOutputArchive &ar) const
{
SerializableQueryCondition cond = *condition_operands.begin();
Condition condition_type = cond.getConditionType();
string condition_str = convertConditionTypeToString(condition_type);
string filter_key = cond.getKey();
string filter_value = cond.getValue();
ar(
cereal::make_nvp("operator", condition_str),
cereal::make_nvp("key", filter_key),
cereal::make_nvp("value", filter_value)
);
condition_operands.begin()->save(ar);
}
void
@@ -101,7 +125,7 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
}
}
const string &
Maybe<SerializableQueryCondition::ValueVariant>
SerializableQueryFilter::getConditionValueByKey(const string &key) const
{
for (const SerializableQueryCondition &condition : condition_operands) {
@@ -110,21 +134,49 @@ SerializableQueryFilter::getConditionValueByKey(const string &key) const
}
}
static string empty_str = "";
return empty_str;
return genError("Key not found.");
}
bool
SerializableQueryFilter::isOperatorComp(const Operator &oper) const
{
return operator_type == Operator::NONE || operator_type == oper;
}
SerializableQueryFilter
SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type)
SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &oper)
{
SerializableQueryFilter query_filter_res;
vector<SerializableQueryFilter> new_queries_operands;
new_queries_operands.push_back(*this);
new_queries_operands.push_back(other_query);
query_filter_res.operator_type = oper;
if (isOperatorComp(oper) && other_query.isOperatorComp(oper)) {
size_t queries_size = queries_operands.size() + other_query.queries_operands.size();
size_t conditions_size = condition_operands.size() + other_query.condition_operands.size();
query_filter_res.queries_operands.reserve(queries_size);
query_filter_res.condition_operands.reserve(conditions_size);
for (const auto &subquery : queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
for (const auto &subquery : other_query.queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : other_query.condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
} else {
query_filter_res.queries_operands.reserve(2);
query_filter_res.queries_operands.push_back(*this);
query_filter_res.queries_operands.push_back(other_query);
}
query_filter_res.queries_operands = new_queries_operands;
query_filter_res.operator_type = operator_type;
return query_filter_res;
}

View File

@@ -58,6 +58,18 @@ QueryRequest::QueryRequest(
full_response = full_reponse;
}
QueryRequest::QueryRequest(
Condition condition_type,
const string &key,
const int64_t &value,
bool full_reponse,
AttributeKeyType attribute_type
) {
query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
assets_limit = default_assets_limit;
full_response = full_reponse;
}
Maybe<string>
QueryRequest::convertObjectTypeToString() const
{
@@ -139,6 +151,16 @@ QueryRequest::addCondition (
query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
}
void
QueryRequest::addCondition (
Condition condition_type,
const string &key,
const int64_t &value,
AttributeKeyType attribute_type
) {
query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
}
void
QueryRequest::setRequestedAttr(const string &attr, AttributeKeyType attr_type)
{