github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-30 03:34:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb_06_2024-Dev

Browse Source
This commit is contained in:
Ned Wright
2024-02-06 16:41:40 +00:00
parent 9f8535c0f7
commit 623951a2f0
59 changed files with 2207 additions and 1168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

329
core/include/services_sdk/interfaces/i_intelligence_is_v2.h
View File

@@ -17,17 +17,16 @@
#include <chrono>
#include <string>
#include "maybe_res.h"
#include "i_messaging.h"
#include "i_time_get.h"
#include "i_mainloop.h"
#include "intelligence_is_v2/asset_reply.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_is_v2/intelligence_types_v2.h"
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "config.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "maybe_res.h"
namespace Intelligence {
class Invalidation;
class Response;
} // namespace Intelligence
@@ -40,315 +39,27 @@ public:
const std::function<void(const Intelligence::Invalidation &)> &callback
) = 0;
virtual void unregisterInvalidation(uint id) = 0;
virtual Maybe<Intelligence::Response>
getResponse(
const std::vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_bulk
) const = 0;
virtual Maybe<Intelligence::Response> getResponse(const QueryRequest &query_request, bool is_pretty) const = 0;
template<typename Data>
Maybe<std::vector<AssetReply<Data>>>
queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true)
{
uint assets_limit = query_request.getAssetsLimit();
static const uint upper_assets_limit = 50;
if (assets_limit == 0 || assets_limit > upper_assets_limit) {
return genError("Assets limit must be in the range of [1, " + std::to_string(upper_assets_limit) + "]");
}
static const uint upper_confidence_limit = 1000;
bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
if (!min_conf_res) {
return genError(
"Minimum confidence value must be in the range of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
if (query_request.isPagingActivated() && query_request.isPagingFinished()) {
return genError("Paging is activated and already finished. No need for more queries.");
}
IntelligenceQuery<Data> intelligence_query(query_request, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/query";
bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit);
if (!res) {
dbgTrace(D_INTELLIGENCE) << "Could not message fog, trying to get offline intelligence.";
Maybe<std::string> offline_res = getOfflineInfoString(query_request.getQuery());
if (!offline_res.ok()) {
dbgDebug(D_INTELLIGENCE) << "Offline intelligence error: " << offline_res.getErr();
return genError("Could not query intelligence");
}
if (!intelligence_query.loadJson(offline_res.unpack())) {
dbgWarning(D_INTELLIGENCE) << "Offline intelligence error: invalid JSON for requested asset";
return genError("Could not query intelligence");
}
}
if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return genError("Query intelligence response with InProgress status");
}
return intelligence_query.getData();
}
queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true);
template<typename Data>
Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true)
{
static const uint upper_assets_limit = 50;
static const uint upper_confidence_limit = 1000;
for (QueryRequest &query_request : query_requests) {
uint assets_limit = query_request.getAssetsLimit();
if (assets_limit == 0 || assets_limit > upper_assets_limit) {
dbgTrace(D_INTELLIGENCE)
<< "Assets limit for request is "
<< upper_assets_limit
<< ", requests assets: "
<< assets_limit;
return genError("Assets limit valid range is of [1, " + std::to_string(upper_assets_limit) + "]");
}
queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true);
bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
if (!min_conf_res) {
dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
return genError(
"Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
}
IntelligenceQuery<Data> intelligence_query(query_requests, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/queries";
dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items";
bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, upper_assets_limit);
if (!res) {
dbgTrace(D_INTELLIGENCE) << "Could not message fog, bulk request failed.";
return genError("Could not query intelligence");
}
return intelligence_query.getBulkData();
}
private:
template<typename Data>
bool
sendMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip,
uint server_port
) {
if (ip == "" && server_port == 0) {
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
intelligence_query,
MessageCategory::INTELLIGENCE
);
return req_status.ok();
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< ip
<< " port: "
<< server_port
<< " query_uri: "
<< query_uri;
MessageMetadata req_md(ip, server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
intelligence_query,
MessageCategory::INTELLIGENCE,
req_md
);
return req_status.ok();
}
template<typename Data>
bool
sendQueryMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip = "",
uint server_port = 0
) {
auto i_timer = getTimer();
auto i_mainloop = getMainloop();
uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
20,
"intelligence",
"request overall timeout"
);
uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
5,
"intelligence",
"request lap timeout"
);
std::chrono::seconds request_overall_timeout = std::chrono::seconds(request_overall_timeout_conf);
std::chrono::seconds request_lap_timeout = std::chrono::seconds(request_lap_timeout_conf);
std::chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
std::chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
std::chrono::seconds seconds_since_start = std::chrono::seconds(0);
std::chrono::seconds seconds_since_last_lap = std::chrono::seconds(0);
bool res = true;
while (res &&
intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS &&
seconds_since_start < request_overall_timeout &&
seconds_since_last_lap < request_lap_timeout
) {
res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
if (res && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
i_mainloop->yield(true);
}
seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - send_request_start_time
);
seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - last_lap_time
);
last_lap_time = i_timer->getMonotonicTime();
}
return res;
}
template<typename Data>
bool
sendPagingQueryMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
int assets_limit,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip = "",
uint server_port = 0
) {
bool res= true;
res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
if (intelligence_query.getResponseStatus() == ResponseStatus::DONE &&
intelligence_query.getResponseAssetCollectionsSize() < assets_limit
) {
intelligence_query.setRequestCursor(Intelligence_IS_V2::CursorState::DONE, "");
} else {
intelligence_query.setRequestCursor(
Intelligence_IS_V2::CursorState::IN_PROGRESS,
intelligence_query.getResponseCursorVal()
);
}
return res;
}
// LCOV_EXCL_START Reason: one templated instance is tested in intelligence ut. the rest are tested in system tests
template<typename Data>
bool
sendQueryObjectToLocalServer(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
const std::string &ip,
bool is_primary_port,
int assets_limit,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags
) {
static const std::string primary_port_setting = "local intelligence server primary port";
static const std::string secondary_port_setting = "local intelligence server secondary port";
auto server_port = getSetting<uint>(
"intelligence",
is_primary_port ? primary_port_setting : secondary_port_setting
);
if (!server_port.ok()) return false;
conn_flags.reset();
if (intelligence_query.getPagingStatus().ok()) {
return sendPagingQueryMessage(
intelligence_query,
query_uri,
assets_limit,
i_message,
conn_flags,
ip,
*server_port
);
}
return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags, ip, *server_port);
}
// LCOV_EXCL_STOP
template<typename Data>
bool
sendQueryObject(IntelligenceQuery<Data> &intelligence_query, const std::string &query_uri, int assets_limit)
{
auto i_message = getMessaging();
Flags<MessageConnectionConfig> conn_flags;
bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
false;
crowdsec_enabled = getProfileAgentSettingWithDefault<bool>(
crowdsec_enabled,
"layer7AccessControl.crowdsec.enabled"
);
bool use_local_intelligence = getProfileAgentSettingWithDefault<bool>(
false,
"agent.config.useLocalIntelligence"
);
auto server_ip = getSetting<std::string>("intelligence", "local intelligence server ip");
if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) {
if (sendQueryObjectToLocalServer(
intelligence_query,
query_uri,
*server_ip,
true,
assets_limit,
i_message,
conn_flags
)
) {
return true;
}
if (sendQueryObjectToLocalServer(
intelligence_query,
query_uri,
*server_ip,
false,
assets_limit,
i_message,
conn_flags
)
) {
return true;
};
}
if (intelligence_query.getPagingStatus().ok()) {
return sendPagingQueryMessage(intelligence_query, query_uri, assets_limit, i_message, conn_flags);
}
return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags);
}
virtual I_Messaging * getMessaging() const = 0;
virtual I_TimeGet * getTimer() const = 0;
virtual I_MainLoop * getMainloop() const = 0;
virtual Maybe<std::string> getOfflineInfoString(const SerializableQueryFilter &query) const = 0;
virtual bool getIsOfflineOnly() const = 0;
protected:
virtual ~I_Intelligence_IS_V2() {}
};
#include "intelligence_is_v2/intelligence_interface_impl.h"
#endif // __I_INTELLIGENCE_IS_V2_H__

99
core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h
View File

@@ -11,18 +11,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __QUERY_RESPONSE_V2_IMPL_H_
#define __QUERY_RESPONSE_V2_IMPL_H_
#ifndef __ASSET_REPLAY_IMPL_H__
#define __ASSET_REPLAY_IMPL_H__
#ifndef __QUERY_RESPONSE_V2_H__
#error intelligence_query_response_v2_impl.h should not be included directly!
#endif // __QUERY_RESPONSE_V2_H__
#ifndef __ASSET_REPLY_H__
#error asset_replay_impl.h should not be included directly!
#endif // __ASSET_REPLY_H__
#include "debug.h"
#include "customized_cereal_multimap.h"
#include "intelligence_types_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
void
AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
@@ -84,87 +82,72 @@ AssetReply<UserSerializableReplyAttr>::getData() const
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::loadFromJson(cereal::JSONInputArchive &ar)
template <typename Values>
bool
AssetReply<UserSerializableReplyAttr>::matchValues(const Values &values) const
{
std::string raw_data;
ar(
cereal::make_nvp("status", raw_data),
cereal::make_nvp("totalNumAssets", total_num_assets),
cereal::make_nvp("assetCollections", asset_collections)
);
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
if (source.template matchValues<Values>(values)) return true;
}
return false;
}
try {
ar(cereal::make_nvp("cursor", cursor));
} catch(...) {}
template <typename UserSerializableReplyAttr>
UserSerializableReplyAttr
AssetReply<UserSerializableReplyAttr>::mergeReplyData() const
{
UserSerializableReplyAttr reply_data;
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
UserSerializableReplyAttr data_by_source = source.mergeReplyData();
reply_data.insert(data_by_source);
}
return reply_data;
}
template<typename UserSerializableReplyAttr>
void
IntelligenceQueryResponseT<UserSerializableReplyAttr>::loadFromJson(const std::string &json_response)
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
serialize(in_ar);
}
template<typename UserSerializableReplyAttr>
template<class Archive>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::serialize(Archive &ar)
IntelligenceQueryResponseT<UserSerializableReplyAttr>::serialize(Archive &ar)
{
std::string raw_data;
ar(
cereal::make_nvp("status", raw_data),
cereal::make_nvp("totalNumAssets", total_num_assets),
cereal::make_nvp("assetCollections", asset_collections)
);
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
try {
ar(cereal::make_nvp("cursor", cursor));
IntelligenceQueryResponse::serialize(ar);
} catch(...) {}
}
template <typename UserSerializableReplyAttr>
Intelligence_IS_V2::ResponseStatus
IntelligenceQueryResponse<UserSerializableReplyAttr>::getResponseStatus() const
{
return status;
}
template <typename UserSerializableReplyAttr>
uint
IntelligenceQueryResponse<UserSerializableReplyAttr>::getAmountOfAssets() const
{
return total_num_assets;
}
template <typename UserSerializableReplyAttr>
const std::string &
IntelligenceQueryResponse<UserSerializableReplyAttr>::getCursor() const
{
return cursor;
}
template <typename UserSerializableReplyAttr>
int
IntelligenceQueryResponse<UserSerializableReplyAttr>::getAssetCollectionsSize() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::getAssetCollectionsSize() const
{
return asset_collections.size();
}
template <typename UserSerializableReplyAttr>
const std::vector<AssetReply<UserSerializableReplyAttr>> &
IntelligenceQueryResponse<UserSerializableReplyAttr>::getData() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::getData() const
{
return asset_collections;
}
template <typename UserSerializableReplyAttr>
bool
IntelligenceQueryResponse<UserSerializableReplyAttr>::isValidInBulk() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::isLast(uint asset_limit)
{
return !partial_fail_in_bulk;
return getResponseStatus() == ResponseStatus::DONE && getAssetCollectionsSize() < asset_limit;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::setFailInBulk()
{
partial_fail_in_bulk = true;
}
#endif // __QUERY_RESPONSE_V2_IMPL_H_
#endif // __ASSET_REPLAY_IMPL_H__

81
core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h
View File

@@ -11,25 +11,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __QUERY_RESPONSE_V2_H__
#define __QUERY_RESPONSE_V2_H__
#ifndef __ASSET_REPLY_H__
#define __ASSET_REPLY_H__
#include <sstream>
#include <vector>
#include <map>
#include <string>
#include <vector>
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "cereal/types/map.hpp"
#include "debug.h"
#include "maybe_res.h"
#include "customized_cereal_map.h"
#include "customized_cereal_multimap.h"
#include "asset_source.h"
#include "query_request_v2.h"
#include "intelligence_types_v2.h"
#include "asset_source_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
#include "maybe_res.h"
template <typename UserSerializableReplyAttr>
class AssetReply
@@ -55,26 +47,10 @@ public:
const std::string & getAssetOrder() const { return asset_order; }
const std::string & getAssetKind() const { return asset_kind; }
UserSerializableReplyAttr
mergeReplyData() const
{
UserSerializableReplyAttr reply_data;
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
UserSerializableReplyAttr data_by_source = source.mergeReplyData();
reply_data.insert(data_by_source);
}
return reply_data;
}
UserSerializableReplyAttr mergeReplyData() const;
template <typename Values>
bool
matchValues(const Values &values) const
{
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
if (source.template matchValues<Values>(values)) return true;
}
return false;
}
bool matchValues(const Values &values) const;
private:
uint asset_schema_version = 0;
@@ -94,33 +70,48 @@ private:
std::vector<SerializableAssetSource<UserSerializableReplyAttr>> sources;
};
template <typename UserSerializableReplyAttr>
class IntelligenceQueryResponse
{
public:
IntelligenceQueryResponse() {}
void loadFromJson(cereal::JSONInputArchive &ar);
void loadFromJson(const std::string &json_response);
template<class Archive>
void serialize(Archive &ar);
Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
uint getAmountOfAssets() const;
const std::string & getCursor() const;
int getAssetCollectionsSize() const;
const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
bool isValidInBulk() const;
void setFailInBulk();
Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; }
const std::string & getCursor() const { return cursor; }
uint getAmountOfAssets() const { return total_num_assets; }
bool isValidInBulk() const { return !partial_fail_in_bulk; }
void setFailInBulk() { partial_fail_in_bulk = true; }
private:
Intelligence_IS_V2::ResponseStatus status = Intelligence_IS_V2::ResponseStatus::IN_PROGRESS;
uint total_num_assets = 0;
std::string cursor = "";
std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
bool partial_fail_in_bulk = false;
};
#include "query_response_v2_impl.h"
template <typename UserSerializableReplyAttr>
class IntelligenceQueryResponseT : public IntelligenceQueryResponse
{
public:
void loadFromJson(const std::string &json_response);
#endif // __QUERY_RESPONSE_V2_H__
template<class Archive>
void serialize(Archive &ar);
uint getAssetCollectionsSize() const;
bool isLast(uint asset_limit);
const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
private:
std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
};
#include "asset_replay_impl.h"
#endif // __ASSET_REPLY_H__

38
core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h
View File

@@ -11,14 +11,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ASSET_SOURCE_V2_H__
#define __ASSET_SOURCE_V2_H__
#ifndef __ASSET_SOURCE_H__
#define __ASSET_SOURCE_H__
#include <chrono>
#include <string>
#include <vector>
#include "debug.h"
#include "intelligence_types_v2.h"
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "customized_cereal_map.h"
#include "intelligence_types_v2.h"
template <typename UserSerializableReplyAttr>
class SerializableAssetSource
@@ -26,7 +27,26 @@ class SerializableAssetSource
public:
SerializableAssetSource() {}
void load(cereal::JSONInputArchive &ar);
void load(cereal::JSONInputArchive &ar)
{
uint raw_seconds;
ar(
cereal::make_nvp("tenantId", tenant_id),
cereal::make_nvp("sourceId", source_id),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("ttl", raw_seconds),
cereal::make_nvp("expirationTime", expiration_time),
cereal::make_nvp("confidence", confidence)
);
ttl = std::chrono::seconds(raw_seconds);
UserSerializableReplyAttr raw_attribute;
try {
ar(cereal::make_nvp("attributes", raw_attribute));
attributes.clear();
attributes.push_back(raw_attribute);
} catch(const std::exception &e) {}
}
const std::string & getTenantId() const { return tenant_id; }
const std::string & getSourceId() const { return source_id; }
@@ -66,6 +86,4 @@ private:
std::vector<UserSerializableReplyAttr> attributes;
};
#include "asset_source_v2_impl.h"
#endif //__ASSET_SOURCE_V2_H__
#endif // __ASSET_SOURCE_H__

47
core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h
View File

@@ -1,47 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ASSET_SOURCE_V2_IMPL_H__
#define __ASSET_SOURCE_V2_IMPL_H__
#ifndef __ASSET_SOURCE_V2_H__
#error intelligence_query_impl_8_0.h should not be included directly!
#endif //__ASSET_V2_SOURCE_H__
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
void
SerializableAssetSource<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
{
uint raw_seconds;
ar(
cereal::make_nvp("tenantId", tenant_id),
cereal::make_nvp("sourceId", source_id),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("ttl", raw_seconds),
cereal::make_nvp("expirationTime", expiration_time),
cereal::make_nvp("confidence", confidence)
);
ttl = std::chrono::seconds(raw_seconds);
UserSerializableReplyAttr raw_attribute;
try {
ar(cereal::make_nvp("attributes", raw_attribute));
attributes.clear();
attributes.push_back(raw_attribute);
} catch(const std::exception &e) {}
}
#endif //__ASSET_SOURCE_V2_IMPL_H__

63
core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h
View File

@@ -15,12 +15,12 @@
#define __BULK_QUERY_RESPONSE_V2_H__
#include <sstream>
#include <vector>
#include <string>
#include <vector>
#include "asset_reply.h"
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "debug.h"
#include "intelligence_types_v2.h"
@@ -29,15 +29,7 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
class BulkResponseError
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("statusCode", status_code),
cereal::make_nvp("message", message)
);
}
void serialize(cereal::JSONInputArchive &ar);
unsigned int getIndex() const { return index; }
int getStatusCode() const { return status_code; }
@@ -49,46 +41,69 @@ private:
std::string message;
};
template <typename UserSerializableReplyAttr>
class ValidBulkQueryResponse
{
public:
void serialize(cereal::JSONInputArchive &ar);
unsigned int getIndex() const { return index; }
const IntelligenceQueryResponse & getResponse() const { return response; }
private:
unsigned int index;
IntelligenceQueryResponse response;
};
template <typename UserSerializableReplyAttr>
class ValidBulkQueryResponseT : public ValidBulkQueryResponse
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
try {
ValidBulkQueryResponse::serialize(ar);
} catch (...) {}
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("response", response)
);
}
unsigned int getIndex() const { return index; }
const IntelligenceQueryResponse<UserSerializableReplyAttr> & getResponse() const { return response; }
const IntelligenceQueryResponseT<UserSerializableReplyAttr> & getResponse() const { return response; }
private:
unsigned int index;
IntelligenceQueryResponse<UserSerializableReplyAttr> response;
IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
};
class IntelligenceQueryBulkResponse
{
public:
void serialize(cereal::JSONInputArchive &ar);
const std::vector<ValidBulkQueryResponse> & getValid() { return valid_responses; }
const std::vector<BulkResponseError> & getErrors() { return errors; }
private:
std::vector<ValidBulkQueryResponse> valid_responses;
std::vector<BulkResponseError> errors;
};
template <typename UserSerializableReplyAttr>
class IntelligenceQueryBulkResponse
class IntelligenceQueryBulkResponseT : public IntelligenceQueryBulkResponse
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("queriesResponse", valid_responses));
try {
ar(cereal::make_nvp("errors", errors));
IntelligenceQueryBulkResponse::serialize(ar);
} catch(...) {}
ar(cereal::make_nvp("queriesResponse", valid_responses));
}
const std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
const std::vector<BulkResponseError> & getErrors() { return errors; }
const std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
private:
std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> valid_responses;
std::vector<BulkResponseError> errors;
std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> valid_responses;
};
#endif // __BULK_QUERY_RESPONSE_V2_H__

50
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h Normal file
View File

@@ -0,0 +1,50 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_INTERFACE_IMPL_H__
#define __INTELLIGENCE_INTERFACE_IMPL_H__
#ifndef __I_INTELLIGENCE_IS_V2_H__
#error intelligence_interface_impl.h should not be included directly!
#endif // __I_INTELLIGENCE_IS_V2_H__
template <typename Data>
Maybe<std::vector<AssetReply<Data>>>
I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore_in_progress, bool is_pretty)
{
auto response = getResponse(query_request, is_pretty);
if (!response.ok()) return response.passErr();
auto serializable_response = response->getSerializableResponse<Data>();
if (!query_request.isPagingActivated()) return serializable_response.getData();
if (serializable_response.isLast(query_request.getAssetsLimit())) {
query_request.setCursor(Intelligence_IS_V2::CursorState::DONE, "");
} else {
query_request.setCursor(Intelligence_IS_V2::CursorState::IN_PROGRESS, response->getCursor());
if (ignore_in_progress) return genError("Query intelligence response with InProgress status");
}
return serializable_response.getData();
}
template<typename Data>
Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
I_Intelligence_IS_V2::queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty)
{
auto res = getResponse(query_requests, is_pretty, true);
if (!res.ok()) return res.passErr();
return res->getBulkData<Data>();
}
#endif // __INTELLIGENCE_INTERFACE_IMPL_H__

77
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h
View File

@@ -1,77 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_QUERY_V2_H__
#define __INTELLIGENCE_QUERY_V2_H__
#include<vector>
#include "cereal/archives/json.hpp"
#include "intelligence_types_v2.h"
#include "query_request_v2.h"
#include "query_response_v2.h"
#include "bulk_query_response_v2.h"
#include "rest.h"
template <typename UserSerializableReplyAttr>
class IntelligenceQuery
{
public:
IntelligenceQuery(QueryRequest &filter, bool is_pretty)
:
request(filter),
response(),
responses(),
is_bulk(false),
is_pretty(is_pretty)
{}
IntelligenceQuery(std::vector<QueryRequest> &filters, bool is_pretty)
:
requests(filters),
response(),
responses(),
is_bulk(true),
is_pretty(is_pretty)
{}
Maybe<std::string> genJson() const;
bool loadJson(const std::string &json);
void load(cereal::JSONInputArchive &ar);
void save(cereal::JSONOutputArchive &ar) const;
std::vector<AssetReply<UserSerializableReplyAttr>> getData();
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> getBulkData();
ResponseStatus getResponseStatus();
int getResponseAssetCollectionsSize() const { return response.getAssetCollectionsSize(); }
const std::string & getResponseCursorVal() const { return response.getCursor(); }
void activatePaging();
Maybe<Intelligence_IS_V2::CursorState> getPagingStatus();
void setRequestCursor(CursorState state, const std::string &value);
private:
static QueryRequest dummy_query_request;
static std::vector<QueryRequest> dummy_query_requests;
std::vector<QueryRequest> &requests = dummy_query_requests;
QueryRequest &request = dummy_query_request;
IntelligenceQueryResponse<UserSerializableReplyAttr> response;
std::vector<IntelligenceQueryResponse<UserSerializableReplyAttr>> responses;
bool is_bulk;
bool is_pretty;
};
#include "intelligence_query_v2_impl.h"
#endif // __INTELLIGENCE_QUERY_V2_H__

191
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h
View File

@@ -1,191 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_QUERY_V2_IMPL_H_
#define __INTELLIGENCE_QUERY_V2_IMPL_H_
#ifndef __INTELLIGENCE_QUERY_V2_H__
#error intelligence_query_impl_v2.h should not be included directly!
#endif // __INTELLIGENCE_QUERY_V2_H__
#include <sstream>
#include "json_stream.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
QueryRequest IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_request = QueryRequest();
template <typename UserSerializableReplyAttr>
std::vector<QueryRequest> IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_requests =
std::vector<QueryRequest>();
template <typename UserSerializableReplyAttr>
Maybe<std::string>
IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
{
{
std::stringstream str_stream;
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(json_stream);
if (is_bulk) {
std::vector<BulkQueryRequest> bulk_requests;
int index = 0;
for (QueryRequest &request : requests) {
bulk_requests.push_back(BulkQueryRequest(request, index++));
}
out_ar(cereal::make_nvp("queries", bulk_requests));
} else {
request.saveToJson(out_ar);
}
}
return str_stream.str();
}
}
template <typename UserSerializableReplyAttr>
bool
IntelligenceQuery<UserSerializableReplyAttr>::loadJson(const std::string &json)
{
try {
std::stringstream in;
in.str(json);
try {
cereal::JSONInputArchive in_ar(in);
load(in_ar);
} catch (const Intelligence_IS_V2::IntelligenceException &e) {
dbgWarning(D_INTELLIGENCE) << "Failed to load query response. Error: " << e.what();
return false;
}
return true;
} catch (const std::exception &e) {
return false;
}
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
{
if (is_bulk) {
IntelligenceQueryBulkResponse<UserSerializableReplyAttr> bulk_response;
bulk_response.serialize(ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
responses.clear();
responses.reserve(requests.size());
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items";
for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) {
if (valid_response[valid_idx].getIndex() == query_idx) {
responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
responses.emplace_back();
responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
responses[query_idx].setFailInBulk();
}
}
} else {
response.loadFromJson(ar);
}
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::save(cereal::JSONOutputArchive &ar) const
{
if (!is_bulk) {
request.saveToJson(ar);
} else {
ar(cereal::make_nvp("queries", requests));
}
}
template <typename UserSerializableReplyAttr>
std::vector<AssetReply<UserSerializableReplyAttr>>
IntelligenceQuery<UserSerializableReplyAttr>::getData()
{
return response.getData();
}
template <typename UserSerializableReplyAttr>
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
IntelligenceQuery<UserSerializableReplyAttr>::getBulkData()
{
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
bulk_data.reserve(responses.size());
int index = 0;
for (const auto &res: responses) {
if (!res.isValidInBulk()) {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
bulk_data.push_back(genError("Received error for request in bulk"));
index++;
} else {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
bulk_data.push_back(res.getData());
index++;
}
}
return bulk_data;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::activatePaging()
{
request.setCursor(Intelligence_IS_V2::CursorState::START, "start");
}
template <typename UserSerializableReplyAttr>
Maybe<Intelligence_IS_V2::CursorState>
IntelligenceQuery<UserSerializableReplyAttr>::getPagingStatus()
{
if (is_bulk) return genError("Paging not activated in bulk mode");
if (!request.isPagingActivated()) return genError("Paging not activated");
return request.getCursorState();
}
template <typename UserSerializableReplyAttr>
ResponseStatus
IntelligenceQuery<UserSerializableReplyAttr>::getResponseStatus()
{
if (!is_bulk) return response.getResponseStatus();
if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
for (const auto &response_itr : responses) {
if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return ResponseStatus::IN_PROGRESS;
}
}
return ResponseStatus::DONE;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::setRequestCursor(CursorState state, const std::string &value)
{
request.setCursor(state, value);
}
#endif //__INTELLIGENCE_QUERY_V2_IMPL_H_

111
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h Normal file
View File

@@ -0,0 +1,111 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_RESPONSE_H__
#define __INTELLIGENCE_RESPONSE_H__
#include <chrono>
#include <string>
#include <vector>
#include <maybe_res.h>
#include "asset_reply.h"
#include "bulk_query_response_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
namespace Intelligence
{
class Response
{
public:
Response() = default;
Response(const std::string &json_body, size_t size, bool is_bulk)
:
json_response(json_body), size(size), is_bulk(is_bulk)
{}
Maybe<void> load();
Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
const std::string getCursor() const { return single_response.getCursor(); }
void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; }
template <typename UserSerializableReplyAttr>
IntelligenceQueryResponseT<UserSerializableReplyAttr> getSerializableResponse() const
{
IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
response.loadFromJson(json_response);
return response;
}
template <typename UserSerializableReplyAttr>
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
getBulkData() const
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
IntelligenceQueryBulkResponseT<UserSerializableReplyAttr> bulk_response;
bulk_response.serialize(in_ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
std::vector<IntelligenceQueryResponseT<UserSerializableReplyAttr>> serializable_responses;
serializable_responses.reserve(size);
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
serializable_responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
serializable_responses.emplace_back();
serializable_responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
serializable_responses[query_idx].setFailInBulk();
}
}
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
bulk_data.reserve(serializable_responses.size());
int index = 0;
for (const auto &res: serializable_responses) {
if (!res.isValidInBulk()) {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
bulk_data.push_back(genError("Received error for request in bulk"));
index++;
} else {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
bulk_data.push_back(res.getData());
index++;
}
}
return bulk_data;
}
private:
std::string json_response;
std::vector<IntelligenceQueryResponse> responses;
IntelligenceQueryResponse single_response;
size_t size = 0;
bool is_bulk = false;
};
}
#endif // __INTELLIGENCE_RESPONSE_H__

3
core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h
View File

@@ -59,7 +59,6 @@ public:
Condition getConditionType() const { return condition_type; }
const std::string & getKey() const { return key; }
const ValueVariant & getValue() const { return value; }
private:
Condition condition_type = Condition::EQUALS;
@@ -85,8 +84,6 @@ public:
const std::vector<SerializableQueryCondition> & getConditionOperands() const { return condition_operands; }
const std::vector<SerializableQueryFilter> & getQueriesOperands() const { return queries_operands; }
Maybe<SerializableQueryCondition::ValueVariant> getConditionValueByKey(const std::string &key) const;
bool empty() const { return condition_operands.empty() && queries_operands.empty(); }
SerializableQueryFilter operator &&(const SerializableQueryFilter &other_query);

21
core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
View File

@@ -97,11 +97,11 @@ public:
void setObjectType(const ObjectType &obj_type);
void setAssetsLimit(uint _assets_limit);
bool checkMinConfidence(uint upper_confidence_limit);
bool checkMinConfidence(uint upper_confidence_limit) const;
void activatePaging();
bool isPagingActivated();
Maybe<CursorState> getCursorState();
Maybe<CursorState> getCursorState() const;
bool isPagingFinished();
void setCursor(CursorState state, const std::string &value);
bool empty() const { return query.empty(); }
@@ -124,21 +124,4 @@ private:
Maybe<std::string> convertObjectTypeToString() const;
};
class BulkQueryRequest
{
public:
BulkQueryRequest() {}
BulkQueryRequest(QueryRequest &request, int index);
void saveToJson(cereal::JSONOutputArchive &ar) const;
void save(cereal::JSONOutputArchive &ar) const;
QueryRequest getQueryRequest() const;
private:
QueryRequest request;
int index;
};
#endif // __QUERY_REQUEST_V2_H__

2
core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h
View File

@@ -51,7 +51,7 @@ public:
uint getSize() const { return requested_attributes.size(); }
bool isRequestedAttributesMapEmpty() const { return requested_attributes.empty(); }
bool checkMinConfidence(uint upper_confidence_limit);
bool checkMinConfidence(uint upper_confidence_limit) const;
private:
std::unordered_map<std::string, uint> requested_attributes;

38
core/include/services_sdk/interfaces/mock/mock_intelligence.h Normal file
View File

@@ -0,0 +1,38 @@
#ifndef __MOCK_INTELLIGENCE_H__
#define __MOCK_INTELLIGENCE_H__
#include "i_intelligence_is_v2.h"
#include "cptest.h"
std::ostream &
operator<<(std::ostream &os, const Intelligence::Response &)
{
return os;
}
std::ostream &
operator<<(std::ostream &os, const Intelligence::Invalidation &)
{
return os;
}
class MockIntelligence : public Singleton::Provide<I_Intelligence_IS_V2>::From<MockProvider<I_Intelligence_IS_V2>>
{
public:
using InvalidationCb = std::function<void(const Intelligence::Invalidation &)>;
using Invalidation = Intelligence::Invalidation;
using Response = Intelligence::Response;
MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
MOCK_METHOD1(unregisterInvalidation, void(uint id));
MOCK_CONST_METHOD3(
getResponse,
Maybe<Response>(const std::vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk)
);
MOCK_CONST_METHOD2(getResponse, Maybe<Response>(const QueryRequest &query_request, bool is_pretty));
MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void));
MOCK_CONST_METHOD1(getOfflineInfoString, Maybe<std::string>(const SerializableQueryFilter &query));
};
#endif // __MOCK_INTELLIGENCE_H__

3
core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h
View File

@@ -14,6 +14,9 @@
#ifndef __DATA_STRING_V2_H__
#define __DATA_STRING_V2_H__
#include <string>
#include "cereal/archives/json.hpp"
class DataString
{
public:

1
core/include/services_sdk/utilities/agent_core_utilities.h
View File

@@ -32,6 +32,7 @@ Maybe<std::vector<std::string>> getDirectoryFiles(const std::string &path);
bool makeDir(const std::string &path, mode_t permission = S_IRWXU);
bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU);
bool deleteDirectory(const std::string &path, bool delete_content = false);
bool touchFile(const std::string &path);
bool
copyFile(