github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb_06_2024-Dev

Browse Source
This commit is contained in:
Ned Wright
2024-02-06 16:41:40 +00:00
parent 9f8535c0f7
commit 623951a2f0
59 changed files with 2207 additions and 1168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
core/intelligence_is_v2/CMakeLists.txt
View File

@@ -1,7 +1,7 @@
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc intelligence_server.cc intelligence_response.cc asset_replay.cc bulk_query_response_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc intelligence_request.cc
)
add_subdirectory(intelligence_is_v2_ut)
#add_subdirectory(intelligence_is_v2_ut)

38
core/intelligence_is_v2/asset_replay.cc Normal file
View File

@@ -0,0 +1,38 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/asset_reply.h"
using namespace std;
void
IntelligenceQueryResponse::loadFromJson(const std::string &json_response)
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
serialize(in_ar);
}
template <class Archive>
void
IntelligenceQueryResponse::serialize(Archive &ar)
{
std::string raw_data;
ar(cereal::make_nvp("status", raw_data), cereal::make_nvp("totalNumAssets", total_num_assets));
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
try {
ar(cereal::make_nvp("cursor", cursor));
} catch (...) {}
}

39
core/intelligence_is_v2/bulk_query_response_v2.cc Normal file
View File

@@ -0,0 +1,39 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/bulk_query_response_v2.h"
using namespace std;
void
BulkResponseError::serialize(cereal::JSONInputArchive &ar)
{
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("statusCode", status_code),
cereal::make_nvp("message", message)
);
}
void
ValidBulkQueryResponse::serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("index", index), cereal::make_nvp("response", response));
}
void
IntelligenceQueryBulkResponse::serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("errors", errors));
ar(cereal::make_nvp("queriesResponse", valid_responses));
}

53
core/intelligence_is_v2/include/intelligence_request.h Normal file
View File

@@ -0,0 +1,53 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_REQUEST_H__
#define __INTELLIGENCE_REQUEST_H__
#include "intelligence_is_v2/query_request_v2.h"
#include <vector>
#include "maybe_res.h"
namespace Intelligence {
class IntelligenceRequest : ClientRest
{
public:
IntelligenceRequest(const std::vector<QueryRequest> &queries, bool is_pretty, bool is_bulk)
:
queries(queries), is_pretty(is_pretty), is_bulk(is_bulk)
{}
Maybe<void> checkAssetsLimit() const;
Maybe<void> checkMinConfidence() const;
bool isPagingAllowed() const;
bool isPagingActivated() const;
Maybe<bool> isPagingFinished() const;
Maybe<Intelligence_IS_V2::CursorState> getPagingStatus() const;
bool loadJson(const std::string &json);
Maybe<std::string> genJson() const;
Maybe<std::string> getResponseFromFog() const;
size_t getSize() const { return queries.size(); }
bool isBulk() const { return is_bulk; }
private:
const std::vector<QueryRequest> &queries;
bool is_pretty = true;
bool is_bulk = false;
Maybe<std::string> response_from_fog = genError("Uninitialized");
};
}
#endif // __INTELLIGENCE_REQUEST_H__

50
core/intelligence_is_v2/include/intelligence_server.h Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_SERVER_H__
#define __INTELLIGENCE_SERVER_H__
#include <vector>
#include "maybe_res.h"
#include "i_messaging.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_request.h"
namespace Intelligence {
class Sender
{
public:
Sender(IntelligenceRequest request);
Maybe<Response> sendIntelligenceRequest();
private:
Maybe<Response> sendQueryObjectToLocalServer(bool is_primary_port);
Maybe<Response> sendQueryMessage();
Maybe<Response> sendMessage();
Maybe<Response> createResponse();
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;
bool is_local_intelligence;
Maybe<std::string> server_ip = genError("No server ip set");
Maybe<unsigned int> server_port = genError("No port is set");
I_Messaging * i_message = nullptr;
I_TimeGet * i_timer = nullptr;
I_MainLoop * i_mainloop = nullptr;
};
}
#endif // __INTELLIGENCE_SERVER_H__

110
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -17,9 +17,10 @@
#include "cache.h"
#include "config.h"
#include "table.h"
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_invalidation.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_request.h"
#include "intelligence_server.h"
using namespace std;
using namespace chrono;
@@ -221,50 +222,10 @@ class IntelligenceComponentV2::Impl
Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
{
public:
class OfflineIntelligeceHandler
{
public:
void
init()
{
filesystem_prefix = getFilesystemPathConfig();
dbgTrace(D_INTELLIGENCE) << "OfflineIntelligeceHandler init. file systen prefix: " << filesystem_prefix;
offline_intelligence_path = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf/offline/intelligence",
"intelligence",
"offline intelligence path"
);
}
Maybe<string>
getValueByIdentifier(const string &identifier) const
{
string asset_file_path = offline_intelligence_path + "/" + identifier;
ifstream asset_info(asset_file_path);
if (!asset_info.is_open()) {
return genError("Could not open file: " + asset_file_path);
}
stringstream info_txt;
info_txt << asset_info.rdbuf();
asset_info.close();
return info_txt.str();
}
private:
string filesystem_prefix = "";
string offline_intelligence_path = "";
};
void
init()
{
offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
registerConfigLoadCb([&]() {
offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
});
offline_intelligence.init();
message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
@@ -283,7 +244,6 @@ public:
bool
sendInvalidation(const Invalidation &invalidation) const override
{
if (offline_mode_only) return false;
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
}
@@ -301,48 +261,28 @@ public:
invalidations.erase(id);
}
I_Messaging *
getMessaging() const override
Maybe<Response>
getResponse(const vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk) const override
{
return message != NULL ? message : Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
}
I_TimeGet *
getTimer() const override
{
return timer != NULL ? timer : Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
}
I_MainLoop *
getMainloop() const override
{
return mainloop != NULL ? mainloop : Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
}
Maybe<string>
getOfflineInfoString(const SerializableQueryFilter &query) const override
{
string ip_attr_key = "mainAttributes.ip";
auto valueVariant = query.getConditionValueByKey(ip_attr_key);
if (!valueVariant.ok()) {
return genError("could not find IP main attribute in the given query.");
}
const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
if (const string* identifier_value = boost::get<string>(&value)) {
if (*identifier_value == "") {
return genError("Could not find IP main attribute in the given query.");
IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk);
if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr();
if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr();
if (intelligence_req.isPagingActivated()) {
auto is_paging_finished = intelligence_req.isPagingFinished();
if (is_paging_finished.ok() && *is_paging_finished) {
return genError("Paging is activated and already finished. No need for more queries.");
}
return offline_intelligence.getValueByIdentifier(*identifier_value);
}
return genError("Value is not of type string.");
Sender intelligence_server(intelligence_req);
auto response = intelligence_server.sendIntelligenceRequest();
return response;
}
bool
getIsOfflineOnly() const override
Maybe<Intelligence::Response>
getResponse(const QueryRequest &query_request, bool is_pretty) const override
{
return offline_mode_only;
vector<QueryRequest> queries = {query_request};
return getResponse(queries, is_pretty, false);
}
private:
@@ -362,7 +302,7 @@ private:
bool
sendLocalInvalidationImpl(const Invalidation &invalidation) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
auto server = getSetting<string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
@@ -435,8 +375,6 @@ private:
bool
sendRegistration(const Invalidation &invalidation) const
{
if (offline_mode_only) return false;
InvalidationRegistration registration;
registration.addInvalidation(invalidation);
@@ -446,7 +384,7 @@ private:
bool
sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
auto server = getSetting<string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
@@ -485,13 +423,11 @@ private:
void
sendReccurringInvalidationRegistration() const
{
if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
if (!hasLocalIntelligence() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}
OfflineIntelligeceHandler offline_intelligence;
bool offline_mode_only = false;
InvalidationCallBack invalidations;
I_Messaging *message = nullptr;
I_TimeGet *timer = nullptr;
@@ -511,8 +447,6 @@ void IntelligenceComponentV2::init() { pimpl->init(); }
void
IntelligenceComponentV2::preload()
{
registerExpectedConfiguration<string>("intelligence", "offline intelligence path");
registerExpectedConfiguration<bool>("intelligence", "offline intelligence only");
registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
registerExpectedSetting<string>("intelligence", "local intelligence server ip");

405
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@@ -1,15 +1,15 @@
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "read_attribute_v2.h"
#include "cptest.h"
#include "singleton.h"
#include "config.h"
#include "config_component.h"
#include "mock/mock_messaging.h"
#include "cptest.h"
#include "mock/mock_intelligence.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_time_get.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_time_get.h"
#include "read_attribute_v2.h"
#include "singleton.h"
using namespace std;
using namespace testing;
@@ -49,9 +49,7 @@ public:
mockRestCall(_, "new-invalidation/source/invalidation", _)
).WillRepeatedly(Return(true));
string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
conf.preload();
intelligence.preload();
intelligence.init();
}
@@ -72,6 +70,31 @@ public:
I_MainLoop::Routine routine;
};
class IntelligenceComponentMockTest : public Test, Singleton::Consume<I_Intelligence_IS_V2>
{
public:
IntelligenceComponentMockTest()
{
debug_output.clear();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
setConfiguration<bool>(false, string("metric"), string("fogMetricSendEnable"));
conf.preload();
}
~IntelligenceComponentMockTest()
{
Debug::setNewDefaultStdout(&cout);
}
::Environment env;
ConfigComponent conf;
stringstream debug_output;
StrictMock<MockIntelligence> intelligence_mock;
};
class Profile
{
public:
@@ -97,6 +120,252 @@ private:
DataString phase;
};
TEST_F(IntelligenceComponentMockTest, getResponseErrorTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
Maybe<Intelligence::Response> res_error = genError("Test error");
EXPECT_CALL(intelligence_mock, getResponse(_, _)
).WillOnce(Return(res_error));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_FALSE(maybe_ans.ok());
}
TEST_F(IntelligenceComponentMockTest, getResponseTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
string response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": \"1.1.1.1\",\n"
" \"phase\": \"testing\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"testing\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
"}\n"
);
Intelligence::Response response(response_str, 1, false);
EXPECT_CALL(intelligence_mock, getResponse(_, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
auto vec = maybe_ans.unpack();
EXPECT_EQ(vec.size(), 1u);
auto iter = vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
}
TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentMockTest>();
vector<QueryRequest> requests;
requests.emplace_back(Condition::EQUALS, "category", "whatever", true);
requests.emplace_back(Condition::EQUALS, "category", "cloud", true);
requests.emplace_back(Condition::EQUALS, "category", "nothing", true);
requests.emplace_back(Condition::EQUALS, "category", "iot", true);
string response_str(
"{\n"
" \"errors\": [\n"
" {\n"
" \"index\": 0,\n"
" \"statusCode\": 400,\n"
" \"message\": \"Bad request. Error: Invalid cursor\"\n"
" },"
" {\n"
" \"index\": 2,\n"
" \"statusCode\": 405,\n"
" \"message\": \"Bad request. Error: Something else\"\n"
" }"
" ],\n" // errors
" \"queriesResponse\": [\n"
" {\n"
" \"index\": 1,\n"
" \"response\": {\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-ip\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"some-group-id\",\n"
" \"name\": \"[1.1.1.1]\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"ip\",\n"
" \"group\": \"\",\n"
" \"order\": \"\",\n"
" \"kind\": \"\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": [\n"
" \"1.1.1.1\",\n"
" \"2.2.2.2\"\n"
" ],\n"
" \"phase\": \"testing\"\n"
" },\n" // mainAttributes
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"color\": \"red\",\n"
" \"user\": \"Omry\",\n"
" \"phase\": \"testing\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" },\n" // source 1
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
" \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 600,\n"
" \"attributes\": {\n"
" \"color\": \"white\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n" // source 2
" ]\n" // sources
" }\n" // asset 1
" ],\n" // asset collection
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
" }\n" // response
" },\n" // queryresponse 1
" {\n"
" \"index\": 3,\n"
" \"response\": {\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-ip\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"some-group-id\",\n"
" \"name\": \"[2.2.2.2]\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"iot\",\n"
" \"family\": \"ip\",\n"
" \"group\": \"\",\n"
" \"order\": \"\",\n"
" \"kind\": \"\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": [\n"
" \"1.1.1.1\",\n"
" \"2.2.2.2\"\n"
" ],\n"
" \"phase\": \"testing\"\n"
" },\n" // mainAttributes
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"color\": \"red\",\n"
" \"user\": \"Omry2\",\n"
" \"phase\": \"testing2\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" },\n" // source 1
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
" \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 600,\n"
" \"attributes\": {\n"
" \"color\": \"white\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n" // source 2
" ]\n" // sources
" }\n" // asset 1
" ],\n" // asset collection
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
" }\n" // response
" }\n" // queryresponse 1
" ]\n" // queryresponses
"}\n"
);
Intelligence::Response response(response_str, 4, true);
EXPECT_CALL(intelligence_mock, getResponse(_, _, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(requests);
EXPECT_TRUE(maybe_ans.ok());
auto vec = maybe_ans.unpack();
EXPECT_EQ(vec.size(), 4u);
EXPECT_FALSE(vec[0].ok());
EXPECT_TRUE(vec[1].ok());
EXPECT_FALSE(vec[2].ok());
EXPECT_TRUE(vec[3].ok());
auto assets1_vec = vec[1].unpack();
EXPECT_EQ(assets1_vec.size(), 1u);
auto iter = assets1_vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
auto assets3_vec = vec[3].unpack();
EXPECT_EQ(assets1_vec.size(), 1u);
iter = assets3_vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
}
TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@@ -154,6 +423,75 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
}
TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
string response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": \"1.1.1.1\",\n"
" \"phase\": \"testing\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"testing\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
"}\n"
);
MessageMetadata md;
EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
EXPECT_EQ(md.getHostName(), "127.0.0.1");
}
TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@@ -626,6 +964,7 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str)));
auto maybe_ans3 = intell->queryIntelligence<Profile>(request);
if (!maybe_ans3.ok()) cout << maybe_ans3.getErr() + "\n";
EXPECT_TRUE(maybe_ans3.ok());
auto vec3 = maybe_ans3.unpack();
EXPECT_EQ(vec3.size(), 1);
@@ -636,54 +975,6 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
}
TEST_F(IntelligenceComponentTestV2, offlineIntelligenceTest)
{
setConfiguration<bool>(true, string("intelligence"), string("offline intelligence only"));
intelligence.init();
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "ip", "1.2.3.4", true);
auto maybe_ans = intell->queryIntelligence<Profile>(request);
ASSERT_TRUE(maybe_ans.ok());
vector<AssetReply<Profile>> vec = maybe_ans.unpack();
vector<AssetReply<Profile>>::iterator assets_iter = vec.begin();
EXPECT_EQ(assets_iter->getAssetSchemaVersion(), 1);
EXPECT_EQ(assets_iter->getAssetType(), "workload-cloud-ip");
EXPECT_EQ(assets_iter->getAssetTypeSchemaVersion(), 1);
EXPECT_EQ(assets_iter->getAssetPermissionGroupId(), "offline-group-id");
EXPECT_EQ(assets_iter->getAssetName(), "offline-asset");
EXPECT_EQ(assets_iter->getAssetClass(), "workload");
EXPECT_EQ(assets_iter->getAssetCategory(), "cloud");
EXPECT_EQ(assets_iter->getAssetFamily(), "offline family");
EXPECT_EQ(assets_iter->getAssetGroup(), "offline testing");
EXPECT_EQ(assets_iter->getAssetOrder(), "");
EXPECT_EQ(assets_iter->getAssetKind(), "");
map<string, vector<string>> attributes_map = assets_iter->getMainAttributes();
EXPECT_EQ(attributes_map["ip"].front(), "1.2.3.4");
vector<SerializableAssetSource<Profile>>::const_iterator sources_iter = assets_iter->getSources().begin();
EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229aa00");
EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00");
EXPECT_EQ(sources_iter->getAssetId(), "50255c3172b4fb7fda93025f0bfaa7abefd1");
EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
EXPECT_EQ(sources_iter->getConfidence(), 700);
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
sources_iter++;
EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229bb11");
EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7bbb11");
EXPECT_EQ(sources_iter->getAssetId(), "cb068860528cb6bfb000cc35e79f11aeefed2");
EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
EXPECT_EQ(sources_iter->getConfidence(), 600);
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Max");
EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
}
TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();

61
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
View File

@@ -11,7 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "intelligence_request.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "cptest.h"
@@ -22,7 +23,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
IntelligenceQuery<int> query(request, true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, true, false);
std::string expected = "{\n"
" \"limit\": 20,\n"
@@ -39,7 +41,9 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
IntelligenceQuery<int> query(request, false);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false);
std::string expected = "{"
"\"limit\":20,"
"\"fullResponse\":true,"
@@ -54,7 +58,8 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) {
QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true);
IntelligenceQuery<int> query(request, false);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false);
std::string expected = "{"
"\"limit\":20,"
"\"fullResponse\":true,"
@@ -71,7 +76,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
IntelligenceQuery<int> query(requests, true);
Intelligence::IntelligenceRequest query(requests, true, true);
std::string expected = "{\n"
" \"queries\": [\n"
@@ -109,30 +114,30 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
IntelligenceQuery<int> query(requests, false);
Intelligence::IntelligenceRequest query(requests, false, true);
std::string expected = "{"
"\"queries\":[{"
"\"query\":{"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
"\"operator\":\"equals\","
"\"key\":\"mainAttributes.phase\","
"\"value\":\"testing\""
"}},"
"\"index\":0"
"},{"
"\"query\":{"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
"\"operator\":\"equals\","
"\"key\":\"mainAttributes.height\","
"\"value\":\"testing\""
"}},"
"\"index\":1"
"}]}";
std::string expected = "{"
"\"queries\":[{"
"\"query\":{"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
"\"operator\":\"equals\","
"\"key\":\"mainAttributes.phase\","
"\"value\":\"testing\""
"}},"
"\"index\":0"
"},{"
"\"query\":{"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
"\"operator\":\"equals\","
"\"key\":\"mainAttributes.height\","
"\"value\":\"testing\""
"}},"
"\"index\":1"
"}]}";
EXPECT_EQ(*query.genJson(), expected);
}

9
core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
View File

@@ -172,6 +172,15 @@ TEST(QueryRequestTestV2, AttributesTest)
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, AttributeConditionExceptionTest)
{
try {
Intelligence_IS_V2::createAttributeString("jey", (Intelligence_IS_V2::AttributeKeyType)5);
} catch (const IntelligenceException &e) {
EXPECT_THAT(string(e.what()), HasSubstr("Received illegal Attribute Type."));
}
}
TEST(QueryRequestTestV2, AndQueryTest)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

42
core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc
View File

@@ -1,5 +1,3 @@
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_is_v2/asset_source_v2.h"
#include "intelligence_is_v2/intelligence_types_v2.h"
#include "intelligence_is_v2/data_string_v2.h"
@@ -31,12 +29,12 @@ public:
try{
ReadAttribute<DataString>("color", data).serialize(ar);
} catch (exception &e) {
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color";
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color. Error:" + string(e.what());
}
try {
ReadAttribute<DataString>("user", data1).serialize(ar);
} catch (const exception &e) {
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user";
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user. Error:" + string(e.what());
}
}
@@ -47,9 +45,6 @@ private:
TEST(QueryResponseTestV2, ReadAttributeTest)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
ReadAttribute<DataString> obj("user", data);
@@ -70,9 +65,6 @@ TEST(QueryResponseTestV2, ReadAttributeTest)
TEST(QueryResponseTestV2, stringData1Test)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
stringData1 obj;
string data_str(
@@ -95,11 +87,9 @@ TEST(QueryResponseTestV2, stringData1Test)
TEST(QueryResponseTestV2, QueryResponseTestV2)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
IntelligenceQueryResponse obj2;
string data_str(
"{\n"
" \"assetCollections\": [\n"
@@ -161,12 +151,14 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
stringstream ss(data_str);
{
cereal::JSONInputArchive ar(ss);
obj.loadFromJson(ar);
obj.loadFromJson(ss.str());
obj2.loadFromJson(ss.str());
}
EXPECT_EQ(obj.getAmountOfAssets(), 2);
EXPECT_EQ(obj2.getAmountOfAssets(), 2);
EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE);
EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE);
EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1);
EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip");
EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1);
@@ -227,11 +219,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
TEST(QueryResponseTestV2, MainAttributesTestV2)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
string string_attribute(
"{\n"
" \"assetCollections\": [\n"
@@ -277,7 +266,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss(string_attribute);
{
cereal::JSONInputArchive ar(ss);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map = obj.getData().begin()->getMainAttributes();
@@ -331,7 +320,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss2(many_strings_attribute);
{
cereal::JSONInputArchive ar(ss2);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map2 = obj.getData().begin()->getMainAttributes();
@@ -390,7 +379,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss3(strings_vector_attribute);
{
cereal::JSONInputArchive ar(ss3);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map3 = obj.getData().begin()->getMainAttributes();
@@ -403,11 +392,8 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
TEST(QueryResponseTestV2, IntelligenceFailTest)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
string status_fail_data_str(
"{\n"
" \"assetCollections\": [\n"
@@ -456,7 +442,7 @@ TEST(QueryResponseTestV2, IntelligenceFailTest)
{
cereal::JSONInputArchive ar(ss);
try {
obj.loadFromJson(ar);
obj.serialize(ar);
} catch (exception &e) {
EXPECT_EQ(e.what(), error_str);
}

128
core/intelligence_is_v2/intelligence_request.cc Normal file
View File

@@ -0,0 +1,128 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_request.h"
#include "debug.h"
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/json_stream.h"
using namespace Intelligence;
using namespace std;
USE_DEBUG_FLAG(D_INTELLIGENCE);
static const unsigned int upper_assets_limit = 50;
static const unsigned int upper_confidence_limit = 1000;
Maybe<void>
IntelligenceRequest::checkAssetsLimit() const
{
for (const QueryRequest &query_request : queries) {
uint assets_limit = query_request.getAssetsLimit();
if (0 < assets_limit && assets_limit <= upper_assets_limit) continue;
dbgTrace(D_INTELLIGENCE)
<< "Assets limit for request is "
<< upper_assets_limit
<< ", requests assets: "
<< assets_limit;
return genError("Assets limit valid range is of [1, " + to_string(upper_assets_limit) + "]");
}
return Maybe<void>();
}
Maybe<void>
IntelligenceRequest::checkMinConfidence() const
{
for (const QueryRequest &query_request : queries) {
if (query_request.checkMinConfidence(upper_confidence_limit)) continue;
dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
return genError(
"Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
return Maybe<void>();
}
bool
IntelligenceRequest::isPagingActivated() const
{
if (!isPagingAllowed()) return false;
return queries.begin()->getCursorState().ok();
}
Maybe<bool>
IntelligenceRequest::isPagingFinished() const
{
if (!isPagingActivated()) return genError("Paging is not activated");
return queries.begin()->getCursorState().unpack() == CursorState::DONE;
}
Maybe<Intelligence_IS_V2::CursorState>
IntelligenceRequest::getPagingStatus() const
{
if (!isPagingAllowed()) return genError("Paging is not allowed");
return queries.begin()->getCursorState();
}
bool
IntelligenceRequest::isPagingAllowed() const
{
if (isBulk()) return false;
return true;
}
Maybe<std::string>
IntelligenceRequest::genJson() const
{
std::stringstream str_stream;
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(json_stream);
if (isBulk()) {
out_ar.setNextName("queries");
out_ar.startNode();
out_ar.makeArray();
uint index = 0;
for (const auto &query : queries) {
out_ar.setNextName(nullptr);
out_ar.startNode();
out_ar.setNextName("query");
out_ar.startNode();
query.saveToJson(out_ar);
out_ar.finishNode();
out_ar(cereal::make_nvp("index", index));
out_ar.finishNode();
index++;
}
out_ar.finishNode();
} else {
queries.begin()->saveToJson(out_ar);
}
}
return str_stream.str();
}
bool
IntelligenceRequest::loadJson(const string &json)
{
response_from_fog = json;
return true;
}
Maybe<string>
IntelligenceRequest::getResponseFromFog() const
{
return response_from_fog;
}

74
core/intelligence_is_v2/intelligence_response.cc Normal file
View File

@@ -0,0 +1,74 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/intelligence_response.h"
using namespace std;
using namespace Intelligence;
Maybe<void>
Response::load()
{
try {
stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
if (is_bulk) {
IntelligenceQueryBulkResponse bulk_response;
bulk_response.serialize(in_ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
responses.clear();
responses.reserve(size);
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
responses.emplace_back();
responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
responses[query_idx].setFailInBulk();
}
}
} else {
single_response.serialize(in_ar);
}
} catch(const std::exception &e) {
return genError("Load common data failed. Error: " + string(e.what()));
}
return {};
}
Intelligence_IS_V2::ResponseStatus
Response::getResponseStatus() const
{
if (!is_bulk) return single_response.getResponseStatus();
if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
for (const auto &response_itr : responses) {
if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return ResponseStatus::IN_PROGRESS;
}
}
return ResponseStatus::DONE;
}

203
core/intelligence_is_v2/intelligence_server.cc Normal file
View File

@@ -0,0 +1,203 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_server.h"
#include "config.h"
#include "debug.h"
#include "enum_array.h"
#include "intelligence_comp_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
using namespace Intelligence;
using namespace std;
static const string query_uri = "/api/v2/intelligence/assets/query";
static const string queries_uri = "/api/v2/intelligence/assets/queries";
static const string primary_port_setting = "local intelligence server primary port";
static const string secondary_port_setting = "local intelligence server secondary port";
Sender::Sender(IntelligenceRequest request) : request(request)
{
i_message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
i_timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
i_mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
false;
if (getProfileAgentSettingWithDefault<bool>(crowdsec_enabled, "layer7AccessControl.crowdsec.enabled")) {
is_local_intelligence = true;
}
if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence")) {
is_local_intelligence = true;
}
auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
if (setting_server_ip.ok()) server_ip = *setting_server_ip;
}
Maybe<Response>
Sender::sendIntelligenceRequest()
{
if (server_ip.ok() && is_local_intelligence) {
auto response = sendQueryObjectToLocalServer(true);
if (response.ok()) return response;
dbgWarning(D_INTELLIGENCE) << "Failed to send query to primary port. Error" << response.getErr();
response = sendQueryObjectToLocalServer(false);
if (response.ok()) return response;
dbgWarning(D_INTELLIGENCE) << "Failed to send query to secondary port. Error" << response.getErr();
}
if (request.getPagingStatus().ok()) {
return sendMessage();
}
return sendQueryMessage();
}
Maybe<Response>
Sender::sendQueryObjectToLocalServer(bool is_primary_port)
{
auto local_port = getSetting<uint>(
"intelligence",
is_primary_port ? primary_port_setting : secondary_port_setting
);
if (!local_port.ok()) return genError(
"Failed to send intelligence query to local server. Config Error number:"
+ to_string(static_cast<uint>(local_port.getErr()))
);
server_port = *local_port;
conn_flags.reset();
auto res = sendQueryMessage();
server_port = genError("port unset after use");
return res;
}
Maybe<Response>
Sender::sendQueryMessage()
{
uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
20,
"intelligence",
"request overall timeout"
);
uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
5,
"intelligence",
"request lap timeout"
);
chrono::seconds request_overall_timeout = chrono::seconds(request_overall_timeout_conf);
chrono::seconds request_lap_timeout = chrono::seconds(request_lap_timeout_conf);
chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
chrono::seconds seconds_since_start = chrono::seconds(0);
chrono::seconds seconds_since_last_lap = chrono::seconds(0);
Maybe<Response> res = genError("Uninitialized");
do {
res = sendMessage();
if (res.ok() && res->getResponseStatus() == ResponseStatus::IN_PROGRESS) {
i_mainloop->yield(true);
}
seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - send_request_start_time
);
seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - last_lap_time
);
last_lap_time = i_timer->getMonotonicTime();
} while (res.ok() &&
res->getResponseStatus() == ResponseStatus::IN_PROGRESS &&
seconds_since_start < request_overall_timeout &&
seconds_since_last_lap < request_lap_timeout
);
return res;
}
Maybe<Response>
Sender::sendMessage()
{
if (server_ip.ok() || server_port.ok()) {
if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set");
if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set");
} else if (!server_ip.ok() && !server_port.ok()) {
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE
);
if (req_status.ok()) {
return createResponse();
};
auto response_error = req_status.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ " "
+ req_status.getErr().toString()
);
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< *server_ip
<< " port: "
<< *server_port
<< " query_uri: "
<< (request.isBulk() ? queries_uri : query_uri);
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
request,
MessageCategory::INTELLIGENCE,
req_md
);
if (!req_status.ok()) {
auto response_error = req_status.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ " "
+ req_status.getErr().toString()
);
};
return createResponse();
}
Maybe<Response>
Sender::createResponse()
{
auto mb_json_body = request.getResponseFromFog();
if (!mb_json_body.ok()) return mb_json_body.passErr();
Response response(*mb_json_body, request.getSize(), request.isBulk());
auto load_status = response.load();
if (!load_status.ok()) return load_status.passErr();
return response;
}

12
core/intelligence_is_v2/query_filter_v2.cc
View File

@@ -146,18 +146,6 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
}
}
Maybe<SerializableQueryCondition::ValueVariant>
SerializableQueryFilter::getConditionValueByKey(const string &key) const
{
for (const SerializableQueryCondition &condition : condition_operands) {
if (condition.getConditionType() == Condition::EQUALS && condition.getKey() == key) {
return condition.getValue();
}
}
return genError("Key not found.");
}
bool
SerializableQueryFilter::isOperatorComp(const Operator &oper) const
{

42
core/intelligence_is_v2/query_request_v2.cc
View File

@@ -25,27 +25,6 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
static const EnumArray<ObjectType, string> object_type_to_string_array{"asset", "zone", "configuration", "shortLived"};
BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index)
:
request(_request),
index(_index)
{}
QueryRequest
BulkQueryRequest::getQueryRequest() const
{
return request;
}
void
BulkQueryRequest::save(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("query", getQueryRequest()),
cereal::make_nvp("index", index)
);
}
QueryRequest::QueryRequest(
Condition condition_type,
const string &key,
@@ -96,22 +75,7 @@ QueryRequest::convertObjectTypeToString() const
void
QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("limit", assets_limit),
cereal::make_nvp("fullResponse", full_response),
cereal::make_nvp("query", query)
);
auto objTypeString = convertObjectTypeToString();
if (objTypeString.ok()) {
ar(cereal::make_nvp("objectType", *objTypeString));
} else {
dbgTrace(D_INTELLIGENCE) << objTypeString.getErr();
}
if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second));
requested_attributes.save(ar);
query_types.save(ar);
save(ar);
}
void
@@ -214,7 +178,7 @@ QueryRequest::setAssetsLimit(uint _assets_limit)
}
bool
QueryRequest::checkMinConfidence(uint upper_confidence_limit)
QueryRequest::checkMinConfidence(uint upper_confidence_limit) const
{
return requested_attributes.checkMinConfidence(upper_confidence_limit);
}
@@ -232,7 +196,7 @@ QueryRequest::isPagingActivated()
}
Maybe<CursorState>
QueryRequest::getCursorState()
QueryRequest::getCursorState() const
{
if (!cursor.ok()) return genError("Paging not activated");
return cursor.unpack().first;

2
core/intelligence_is_v2/requested_attributes_v2.cc
View File

@@ -54,7 +54,7 @@ SerializableAttributesMap::getAttributeByKey(const string &key) const
}
bool
SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit)
SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit) const
{
for (auto const &attribute : requested_attributes) {
if (attribute.second == 0 || attribute.second > upper_confidence_limit) return false;