github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jul 5th update

Browse Source
This commit is contained in:
Ned Wright 2023-07-05 23:32:39 +00:00
parent 22f1a984aa
commit a59f079ef7
85 changed files with 2488 additions and 1754 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
attachments/kernel_modules/core/include/common_is/kdebug_flags.h
View File

@ -31,5 +31,6 @@ DEFINE_KDEBUG_FLAG(statefulValidation)
DEFINE_KDEBUG_FLAG(statelessValidation)
DEFINE_KDEBUG_FLAG(kernelMetric)
DEFINE_KDEBUG_FLAG(tproxy)
DEFINE_KDEBUG_FLAG(tenantStats)
#endif // DEFINE_KDEBUG_FLAG

1
components/CMakeLists.txt
View File

@ -5,7 +5,6 @@ add_subdirectory(signal_handler)
add_subdirectory(gradual_deployment)
add_subdirectory(packet)
add_subdirectory(pending_key)
add_subdirectory(messaging_downloader)
add_subdirectory(health_check_manager)
add_subdirectory(utils)

4
components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc
View File

@ -52,14 +52,14 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da
ctx.registerValue("eventReferenceId", uuid, EnvKeyAttr::LogSection::DATA);
ctx.registerValue<string>(HttpTransactionData::http_proto_ctx, transaction_data.getHttpProtocol());
ctx.registerValue<string>(HttpTransactionData::method_ctx, transaction_data.getHttpMethod());
ctx.registerValue<string>(HttpTransactionData::host_name_ctx, transaction_data.getDestinationHost());
ctx.registerValue<string>(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost());
ctx.registerValue<uint16_t>(HttpTransactionData::listening_port_ctx, transaction_data.getListeningPort());
ctx.registerValue<IPAddr>(HttpTransactionData::listening_ip_ctx, transaction_data.getListeningIP());
ctx.registerValue<IPAddr>(HttpTransactionData::client_ip_ctx, transaction_data.getSourceIP());
ctx.registerValue<uint16_t>(HttpTransactionData::client_port_ctx, transaction_data.getSourcePort());
ctx.registerFunc<string>(HttpTransactionData::source_identifier, [this](){ return source_identifier; });
ctx.registerValue<string>(HttpTransactionData::uri_ctx, transaction_data.getURI());
ctx.registerValue<string>(HttpTransactionData::uri_ctx, transaction_data.getParsedURI());
auto decoder = makeVirtualContainer<HexDecoder<'%'>>(transaction_data.getURI());
string decoded_url(decoder.begin(), decoder.end());
auto question_mark_location = decoded_url.find('?');

21
components/include/http_transaction_data.h
View File

@ -43,6 +43,19 @@ public:
uint16_t client_port
);
HttpTransactionData (
std::string http_proto,
std::string method,
std::string host_name,
std::string parsed_host,
IPAddr listening_ip,
uint16_t listening_port,
std::string uri,
std::string parsed_uri,
IPAddr client_ip,
uint16_t client_port
);
// LCOV_EXCL_START - sync functions, can only be tested once the sync module exists
template <class Archive>
void
@ -52,9 +65,11 @@ public:
http_proto,
method,
host_name,
parsed_host,
listening_ip,
listening_port,
uri,
parsed_uri,
client_ip,
client_port,
response_content_encoding
@ -69,9 +84,11 @@ public:
http_proto,
method,
host_name,
parsed_host,
listening_ip,
listening_port,
uri,
parsed_uri,
client_ip,
client_port,
response_content_encoding
@ -86,8 +103,10 @@ public:
const IPAddr & getListeningIP() const { return listening_ip; }
uint16_t getListeningPort() const { return listening_port; }
const std::string & getDestinationHost() const { return host_name; }
const std::string & getParsedHost() const { return parsed_host; }
const std::string & getHttpProtocol() const { return http_proto; }
const std::string & getURI() const { return uri; }
const std::string & getParsedURI() const { return parsed_uri; }
const std::string & getHttpMethod() const { return method; }
void print(std::ostream &out_stream) const;
@ -124,9 +143,11 @@ private:
std::string http_proto;
std::string method = "GET";
std::string host_name;
std::string parsed_host;
IPAddr listening_ip;
uint16_t listening_port;
std::string uri;
std::string parsed_uri;
IPAddr client_ip;
uint16_t client_port;
bool is_request;

2
components/include/i_service_controller.h
View File

@ -51,6 +51,8 @@ public:
virtual void clearFailedServices() = 0;
virtual std::set<std::string> && moveChangedPolicies() = 0;
virtual bool isServiceInstalled(const std::string &service_name) = 0;
virtual void registerServiceConfig(

2
components/include/orchestration_comp.h
View File

@ -14,7 +14,6 @@
#ifndef __ORCHESTRATION_COMP_H__
#define __ORCHESTRATION_COMP_H__
#include "i_messaging_downloader.h"
#include "i_messaging.h"
#include "i_mainloop.h"
#include "i_shell_cmd.h"
@ -49,7 +48,6 @@ class OrchestrationComp
Singleton::Consume<I_DetailsResolver>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_TenantManager>,
Singleton::Consume<I_MessagingDownloader>,
Singleton::Consume<I_PackageHandler>,
Singleton::Consume<I_ServiceController>,
Singleton::Consume<I_UpdateCommunication>,

4
components/messaging_downloader/CMakeLists.txt
View File

@ -1,4 +0,0 @@
add_library(messaging_downloader_server messaging_downloader_server.cc)
add_library(messaging_downloader_client messaging_downloader_client.cc)
add_subdirectory(messaging_downloader_ut)

230
components/messaging_downloader/messaging_downloader_client.cc
View File

@ -1,230 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <unordered_map>
#include "messaging_downloader_client.h"
#include "i_messaging.h"
#include "config.h"
#include "rest.h"
USE_DEBUG_FLAG(D_COMMUNICATION);
using namespace std;
class MessagingDownloaderClientRequest : public ClientRest
{
public:
MessagingDownloaderClientRequest()
{
file_name = "";
url = "";
port = 80;
response_port = 0;
status = false;
}
MessagingDownloaderClientRequest(
const string &_file_name,
const string &_url,
const unsigned int _port,
const unsigned int _response_port
) :
file_name(_file_name),
url(_url),
port(_port),
response_port(_response_port),
status(false)
{}
bool getStatus() const { return status.get(); }
const string & getUuid() const { return uuid.get(); }
C2S_PARAM(string, file_name);
C2S_PARAM(string, url);
C2S_PARAM(unsigned int, port);
C2S_PARAM(unsigned int, response_port);
S2C_PARAM(string, uuid);
S2C_PARAM(bool, status);
};
class DownloaderCbHandler
{
public:
void
addCallback(const string &uuid, I_MessagingDownloader::OnCompleteCB &cb)
{
DownloaderCbHandler::uuid_to_cb[uuid] = cb;
}
static void
handleDownloadCB(const string &uuid, Maybe<string> &downloaded_file)
{
dbgTrace(D_COMMUNICATION) << "Handling downloading complete callback. UUID: " << uuid;
if(DownloaderCbHandler::uuid_to_cb.find(uuid) == DownloaderCbHandler::uuid_to_cb.end()) {
dbgWarning(D_COMMUNICATION) << "Failed to execute download completion callback.";
return;
}
if (DownloaderCbHandler::uuid_to_cb.at(uuid) != nullptr) {
DownloaderCbHandler::uuid_to_cb.at(uuid)(downloaded_file);
DownloaderCbHandler::uuid_to_cb.erase(uuid);
} else {
string curr_status;
if (downloaded_file.ok()) {
curr_status = ". File path: " + downloaded_file.unpack();
} else {
curr_status = ". Error: " + downloaded_file.getErr();
}
dbgWarning(D_COMMUNICATION)
<< "Illegal download completion callback for downloading process with UUID: "
<< uuid
<< curr_status;
}
dbgTrace(D_COMMUNICATION) << "Successfully handled the downloading complete callback. UUID: " << uuid;
}
static unordered_map<string, I_MessagingDownloader::OnCompleteCB> uuid_to_cb;
};
unordered_map<string, I_MessagingDownloader::OnCompleteCB> DownloaderCbHandler::uuid_to_cb;
class MessagingDownloaderClientRes : public ServerRest
{
public:
void
doCall() override
{
dbgTrace(D_COMMUNICATION) << "Received response from the downloading server.";
if (status.get() && filepath.isActive()) {
Maybe<string> response(filepath.get());
DownloaderCbHandler::handleDownloadCB(uuid.get(), response);
} else {
if (!error.isActive()) error = "unknown error";
dbgWarning(D_COMMUNICATION) << "Failed to download. Error: " << error.get();
Maybe<string> response = genError(error.get());
DownloaderCbHandler::handleDownloadCB(uuid.get(), response);
}
}
C2S_PARAM(string, uuid);
C2S_PARAM(bool, status);
C2S_OPTIONAL_PARAM(string, filepath);
C2S_OPTIONAL_PARAM(string, error);
};
class MessagingDownloaderClient::Impl : Singleton::Provide<I_MessagingDownloader>::From<MessagingDownloaderClient>
{
public:
void
init()
{
i_msg = Singleton::Consume<I_Messaging>::by<MessagingDownloaderClient>();
Singleton::Consume<I_RestApi>::by<MessagingDownloaderClient>()->addRestCall<MessagingDownloaderClientRes>(
RestAction::SHOW,
"download-status"
);
}
void
fini()
{
i_msg = nullptr;
}
bool
downloadFile(
const string &file_name,
const string &url,
I_MessagingDownloader::OnCompleteCB cb = nullptr,
const unsigned int port = 0
) override
{
dbgTrace(D_COMMUNICATION)
<< "Processing new download request."
<< "File name: "
<< file_name
<< "URL: "
<< url;
auto response_port = Singleton::Consume<I_Environment>::by<MessagingDownloaderClient>()->get<int>(
"Listening Port"
);
if (!response_port.ok()) {
dbgWarning(D_COMMUNICATION) << "Failed to get the service listening port.";
return false;
}
vector<int> download_ports = {
getConfigurationWithDefault(8164, "Downloader", "Downloader Primary Port"),
getConfigurationWithDefault(8167, "Downloader", "Downloader Secondary Port")
};
MessagingDownloaderClientRequest download_obj(
file_name,
url,
port,
response_port.unpack()
);
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXPECT_REPLY);
if (i_msg != nullptr) {
dbgTrace(D_COMMUNICATION) << "Sending request to the downloading service.";
bool res = false;
for (int port: download_ports) {
dbgTrace(D_COMMUNICATION) << "Trying to request downloading with downloading service port " << port;
res = i_msg->sendObject(
download_obj,
I_Messaging::Method::POST,
"127.0.0.1",
port,
conn_flags,
"/add-download-file"
);
if (res) break;
}
if (!res) {
dbgInfo(D_COMMUNICATION) << "Failed to request for file downloading";
return false;
}
dbgTrace(D_COMMUNICATION) << "Successfully requested for downloading.";
cb_handler.addCallback(download_obj.getUuid(), cb);
} else {
dbgDebug(D_COMMUNICATION) << "Failed to request downloading. Illegal messaging infrastructure.";
}
return download_obj.getStatus();
}
private:
I_Messaging *i_msg;
DownloaderCbHandler cb_handler;
};
MessagingDownloaderClient::MessagingDownloaderClient()
:
Component("MessagingDownloaderClient"),
pimpl(make_unique<Impl>())
{}
MessagingDownloaderClient::~MessagingDownloaderClient() {}
void MessagingDownloaderClient::init() { pimpl->init(); }
void MessagingDownloaderClient::fini() { pimpl->fini(); }
void
MessagingDownloaderClient::preload()
{
registerExpectedConfiguration<int>("Downloader", "Downloader Primary Port");
registerExpectedConfiguration<int>("Downloader", "Downloader Secondary Port");
};

375
components/messaging_downloader/messaging_downloader_server.cc
View File

@ -1,375 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "messaging_downloader_server.h"
#include <sys/types.h>
#include <sys/stat.h>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include <sstream>
#include <set>
#include "i_messaging.h"
#include "rest.h"
#include "config.h"
#include "url_parser.h"
#include "agent_core_utilities.h"
USE_DEBUG_FLAG(D_COMMUNICATION);
using namespace std;
class MessagingDownloaderResponser : public ClientRest
{
public:
MessagingDownloaderResponser()=delete;
MessagingDownloaderResponser(string &_uuid, const Maybe<string> &_filepath)
:
uuid(_uuid),
status(_filepath.ok())
{
if (_filepath.ok()) {
filepath = _filepath.unpack();
} else {
error = _filepath.getErr();
}
}
C2S_PARAM(string, uuid);
C2S_PARAM(bool, status);
C2S_OPTIONAL_PARAM(string, filepath)
C2S_OPTIONAL_PARAM(string, error)
};
class MessagingDownloaderReceiver : public ServerRest
{
public:
void
doCall() override
{
dbgTrace(D_COMMUNICATION) << "Received new downloading request.";
stringstream uuid_ss;
uuid_ss << boost::uuids::random_generator()();
uuid = uuid_ss.str();
if (!port.isActive()) {
dbgTrace(D_COMMUNICATION) << "Request does not contain explicit port.";
port = 0;
}
dbgInfo(D_COMMUNICATION)
<< "Downloading a file and using the next parameters: "
<< "file_name: "
<< file_name.get()
<< ", url: "
<< url.get()
<< ", uuid: "
<< uuid.get()
<< ", port: "
<< port.get()
<< ", notification port: "
<< response_port.get();
unsigned int response_port_cap = response_port.get();
string uuid_capture = uuid.get();
status = Singleton::Consume<I_MessagingDownloader>::from<MessagingDownloaderServer>()->downloadFile(
file_name.get(),
url.get(),
[uuid_capture, response_port_cap](const Maybe<string> &downloaded_file) mutable
{
Flags<MessageConnConfig> conn_flags;
MessagingDownloaderResponser res(uuid_capture, downloaded_file);
dbgTrace(D_COMMUNICATION) << "Sending the download status to the client.";
bool res_status = Singleton::Consume<I_Messaging>::by<MessagingDownloaderServer>()->sendNoReplyObject(
res,
I_Messaging::Method::POST,
"127.0.0.1",
response_port_cap,
conn_flags,
"/show-download-status"
);
if (!res_status) {
dbgInfo(D_COMMUNICATION) << "Failed to send the download status.";
} else {
dbgDebug(D_COMMUNICATION)
<< "Successfully sent the download status. Notification port: "
<< response_port_cap
<< ", Status: "
<< downloaded_file.ok();
}
},
port.get()
);
}
C2S_PARAM(string, file_name);
C2S_PARAM(string, url);
C2S_PARAM(int, response_port);
C2S_PARAM(int, port);
S2C_PARAM(string, uuid);
S2C_PARAM(bool, status);
};
class DownloadingInstance
{
public:
DownloadingInstance()=default;
DownloadingInstance(
const string &_file_name,
const string &_url,
const unsigned int _port
) :
file_name(_file_name),
url(_url),
port(_port),
url_parser(_url)
{
parseURL();
}
Maybe<string>
genJson() const
{
return string("");
}
bool
loadJson(const string &_body)
{
body = vector<char>(_body.begin(), _body.end());
return true;
}
const vector<char> &
getResponse() const
{
return body;
}
bool
operator==(const DownloadingInstance &other) const
{
return file_name == other.file_name &&
host == other.host &&
url == other.url &&
port == other.port &&
is_secure == other.is_secure &&
origin_is_fog == other.origin_is_fog;
}
bool
operator<(const DownloadingInstance &other) const
{
return file_name < other.file_name ||
host < other.host ||
url < other.url ||
port < other.port ||
is_secure < other.is_secure ||
origin_is_fog < other.origin_is_fog;
}
const string & getFileName() const { return file_name; }
const string & getHost() const { return host; }
const string & getUrl() const { return url; }
unsigned int getPort() const { return port; }
bool getIsSecure() const { return is_secure; }
bool getIsFogOrigin() const { return origin_is_fog; }
private:
void
parseURL()
{
dbgTrace(D_COMMUNICATION) << "Parsing the URL to extract the relevant info. URL: " << url;
origin_is_fog = false;
auto maybe_host = url_parser.getBaseURL();
if (!maybe_host.ok()) {
dbgWarning(D_COMMUNICATION) << "Failed to parse the URL";
return;
}
host = maybe_host.unpack();
is_secure = url_parser.isOverSSL();
if (port == 0 && url_parser.getPort() != "") {
try {
port = stoi(url_parser.getPort());
} catch (exception &e) {
port = 443;
dbgInfo(D_COMMUNICATION)
<< "Failed to parse the port for the downloading request. Error "
<< e.what()
<< ". Using the default port "
<< port;
}
} else {
dbgTrace(D_COMMUNICATION) << "Using explicitly defined port. Port: " << port;
}
I_AgentDetails *agent_details = Singleton::Consume<I_AgentDetails>::by<MessagingDownloaderServer>();
if (agent_details->getFogDomain().ok()) {
string fog_domain = agent_details->getFogDomain().unpack();
if (host.find(fog_domain) != string::npos) {
origin_is_fog = true;
}
} else {
dbgTrace(D_COMMUNICATION) << "Failed to receive fog domain.";
}
}
string file_name = "";
string url = "";
unsigned int port = 0;
URLParser url_parser;
vector<char> body = {};
string host = "";
bool is_secure = true;
bool origin_is_fog = true;
};
class MessagingDownloaderServer::Impl : Singleton::Provide<I_MessagingDownloader>::From<MessagingDownloaderServer>
{
public:
void
init()
{
i_msg = Singleton::Consume<I_Messaging>::by<MessagingDownloaderServer>();
i_mainloop = Singleton::Consume<I_MainLoop>::by<MessagingDownloaderServer>();
auto rest = Singleton::Consume<I_RestApi>::by<MessagingDownloaderServer>();
rest->addRestCall<MessagingDownloaderReceiver>(RestAction::ADD, "download-file");
string default_downloading_dir = "/tmp/cp_nano_downloader/";
download_dir = getConfigurationWithDefault(
default_downloading_dir,
"Downloader",
"Downloading Directory"
);
NGEN::Filesystem::makeDirRecursive(download_dir);
}
void
fini()
{
i_msg = nullptr;
i_mainloop = nullptr;
}
bool
downloadFile(
const string &file_name,
const string &url,
OnCompleteCB on_complete_func = nullptr,
const unsigned int port = 443
) override
{
dbgTrace(D_COMMUNICATION) << "Handling new download request. URL: " << url << ". File name: " << file_name;
DownloadingInstance req(file_name, url, port);
if (downloading_queue.find(req) != downloading_queue.end()) {
dbgInfo(D_COMMUNICATION) << "Failed to download the file. Similar download request already exists.";
return false;
}
if (!isValidPath(file_name)) {
dbgInfo(D_COMMUNICATION) << "Failed to validate the download path. Path: " << download_dir + file_name;
return false;
}
downloading_queue.insert(req);
i_mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::RealTime,
[this, req, on_complete_func]() mutable
{
Flags<MessageConnConfig> conn_flags;
if (req.getIsSecure()) conn_flags.setFlag(MessageConnConfig::SECURE_CONN);
if (!req.getIsFogOrigin()) conn_flags.setFlag(MessageConnConfig::EXTERNAL);
auto on_exit = make_scope_exit([this, &req]() { downloading_queue.erase(req); } );
bool response = i_msg->sendObject(
req,
I_Messaging::Method::GET,
req.getHost(),
req.getPort(),
conn_flags,
req.getUrl()
);
if (response) {
dbgTrace(D_COMMUNICATION) << "Successfully received a response from the downloading file host.";
std::ofstream downloaded_file;
downloaded_file.open(download_dir + req.getFileName());
if (!downloaded_file.is_open()) {
dbgInfo(D_COMMUNICATION)
<< "Failed to download file. Error: Failed to open the file "
<< req.getFileName();
Maybe<string> err = genError("Failed to open the file");
on_complete_func(err);
if (i_mainloop != nullptr) i_mainloop->yield(true);
}
auto &res_body = req.getResponse();
downloaded_file.write(res_body.data(), res_body.size());
downloaded_file.close();
dbgInfo(D_COMMUNICATION) << "Successfully downloaded the file. File name: " << req.getFileName();
Maybe<string> filepath = download_dir + req.getFileName();
on_complete_func(filepath);
} else {
dbgInfo(D_COMMUNICATION) << "Failed to download file. File name: " << req.getFileName();
Maybe<string> err = genError("Failed during the downloading process.");
on_complete_func(err);
}
},
"Download file routine for '" + file_name + "'",
false
);
return true;
}
private:
bool
isValidPath(const string &file_name)
{
struct stat info;
string file_to_download = download_dir + file_name;
dbgTrace(D_COMMUNICATION) << "Validating the downloading file path. Path: " << file_to_download;
if (stat(download_dir.c_str(), &info) != 0) {
dbgDebug(D_COMMUNICATION) << "Failed to access the downloading directory";
return false;
}
if (stat(file_to_download.c_str(), &info) == 0) {
dbgDebug(D_COMMUNICATION)
<< "The file with the name '"
<< file_name
<< "' is already exist in the downloading directory";
return false;
}
return true;
}
I_Messaging *i_msg;
I_MainLoop *i_mainloop;
string download_dir;
set<DownloadingInstance> downloading_queue;
};
MessagingDownloaderServer::MessagingDownloaderServer()
:
Component("MessagingDownloaderServer"),
pimpl(make_unique<Impl>())
{}
MessagingDownloaderServer::~MessagingDownloaderServer() {}
void MessagingDownloaderServer::init() { pimpl->init(); }
void MessagingDownloaderServer::fini() { pimpl->fini(); }
void
MessagingDownloaderServer::preload()
{
registerExpectedConfiguration<string>("Downloader", "Downloading Directory");
};

2
components/messaging_downloader/messaging_downloader_ut/CMakeLists.txt
View File

@ -1,2 +0,0 @@
add_subdirectory(downloader_server_ut)
add_subdirectory(downloader_client_ut)

9
components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/CMakeLists.txt
View File

@ -1,9 +0,0 @@
link_directories(${BOOST_ROOT}/lib)
include_directories(${CMAKE_SOURCE_DIR}/core/include)
link_directories(${CMAKE_BINARY_DIR}/core/include)
add_unit_test(
downloader_client_ut
"downloader_client_ut.cc"
"singleton;messaging_downloader_client;time_proxy;mainloop;rest;metric;event_is;message;-lboost_context;agent_core_utilities;orchestration_modules;connkey;-lboost_regex;-lboost_filesystem;-lboost_system"
)

113
components/messaging_downloader/messaging_downloader_ut/downloader_client_ut/downloader_client_ut.cc
View File

@ -1,113 +0,0 @@
#include "messaging_downloader_client.h"
#include <boost/filesystem.hpp>
#include "environment.h"
#include "singleton.h"
#include "config.h"
#include "config_component.h"
#include "mainloop.h"
#include "cptest.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_agent_details.h"
#include "mock/mock_time_get.h"
using namespace std;
using namespace testing;
class MessagingDownloaderClientTest : public Test
{
public:
MessagingDownloaderClientTest()
{
EXPECT_CALL(
rest,
mockRestCall(RestAction::SHOW, "download-status", _)
).WillOnce(WithArg<2>(Invoke(this, &MessagingDownloaderClientTest::restHandler)));
EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
Debug::setUnitTestFlag(D_COMMUNICATION, Debug::DebugLevel::TRACE);
Debug::setNewDefaultStdout(&capture_debug);
messaging_downloader.preload();
env.preload();
env.init();
messaging_downloader.init();
}
~MessagingDownloaderClientTest()
{
boost::filesystem::remove_all("/tmp/test_download_dir/");
messaging_downloader.fini();
}
bool
restHandler(const unique_ptr<RestInit> &rest_ptr)
{
rest_handler = rest_ptr->getRest();
return true;
}
unique_ptr<ServerRest> rest_handler;
ostringstream capture_debug;
I_MainLoop::Routine downloading_routine;
MessagingDownloaderClient messaging_downloader;
NiceMock<MockTimeGet> mock_time;
NiceMock<MockAgentDetails> mock_agent_details;
StrictMock<MockMessaging> mock_msg;
StrictMock<MockRestApi> rest;
StrictMock<MockMainLoop> mock_ml;
::Environment env;
ConfigComponent conf;
};
TEST_F(MessagingDownloaderClientTest, do_nothing)
{
}
TEST_F(MessagingDownloaderClientTest, request_download)
{
string file_name = "test_file";
string url = "https://download_test.com/test_download";
Singleton::Consume<I_Environment>::by<MessagingDownloaderClient>()->registerValue<int>("Listening Port", 6464);
stringstream ss;
ss << "{\n \"file_name\": \"" << file_name << "\","
<< "\n \"url\": \"" << url << "\","
<< "\n \"port\": 0,\n \"response_port\": 6464\n}";
EXPECT_CALL(mock_msg, sendMessage(
true,
ss.str(),
I_Messaging::Method::POST,
"127.0.0.1",
8164,
_,
"/add-download-file",
_,
_,
_
)).WillOnce(Return(Maybe<string>(string("{\"uuid\": \"111\", \"status\": true}"))));
bool is_cb_run = false;
bool res = Singleton::Consume<I_MessagingDownloader>::from<MessagingDownloaderClient>()->downloadFile(
file_name,
url,
[&is_cb_run](const Maybe<string>& filepath)
{
is_cb_run = true;
EXPECT_TRUE(filepath.ok());
EXPECT_EQ(filepath.unpack(), "/tmp/test_download_dir/test_file");
}
);
EXPECT_TRUE(res);
stringstream is;
is << "{\"uuid\": \"111\", \"status\": true, \"filepath\": \"/tmp/test_download_dir/test_file\"}";
EXPECT_FALSE(is_cb_run);
rest_handler->performRestCall(is);
EXPECT_TRUE(is_cb_run);
}

9
components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/CMakeLists.txt
View File

@ -1,9 +0,0 @@
link_directories(${BOOST_ROOT}/lib)
include_directories(${CMAKE_SOURCE_DIR}/core/include)
link_directories(${CMAKE_BINARY_DIR}/core/include)
add_unit_test(
downloader_server_ut
"downloader_server_ut.cc"
"singleton;messaging_downloader_server;time_proxy;mainloop;rest;metric;event_is;message;-lboost_context;agent_core_utilities;orchestration_modules;agent_details;connkey;-lboost_regex;-lboost_filesystem;-lboost_system"
)

304
components/messaging_downloader/messaging_downloader_ut/downloader_server_ut/downloader_server_ut.cc
View File

@ -1,304 +0,0 @@
#include "messaging_downloader_server.h"
#include <boost/filesystem.hpp>
#include "environment.h"
#include "singleton.h"
#include "config.h"
#include "config_component.h"
#include "mainloop.h"
#include "cptest.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_agent_details.h"
#include "mock/mock_time_get.h"
using namespace std;
using namespace testing;
class MessagingDownloaderServerTest : public Test
{
public:
MessagingDownloaderServerTest()
{
setConfiguration(string("/tmp/test_download_dir/"), "Downloader", "Downloading Directory");
EXPECT_CALL(
rest,
mockRestCall(RestAction::ADD, "download-file", _)
).WillOnce(WithArg<2>(Invoke(this, &MessagingDownloaderServerTest::restHandler)));
Maybe<string> fog_addr(string("test.fog.com"));
EXPECT_CALL(
mock_agent_details,
getFogDomain()
).WillRepeatedly(Return(fog_addr));
Debug::setUnitTestFlag(D_COMMUNICATION, Debug::DebugLevel::TRACE);
Debug::setNewDefaultStdout(&capture_debug);
messaging_downloader.preload();
messaging_downloader.init();
}
~MessagingDownloaderServerTest()
{
boost::filesystem::remove_all("/tmp/test_download_dir/");
messaging_downloader.fini();
}
bool
restHandler(const unique_ptr<RestInit> &rest_ptr)
{
rest_handler = rest_ptr->getRest();
return true;
}
void
expectRequestSuccess(
string &test_file_name,
string &host,
string &url,
string &uuid,
unsigned int port,
unsigned int response_port,
string &success_msg
)
{
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false)
).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0)));
EXPECT_CALL(
mock_msg,
sendMessage(true, "", I_Messaging::Method::GET, host, port, _, url, _, _, _)
).WillOnce(Return(Maybe<string>(string("test_body"))));
stringstream expected_response;
expected_response
<< "\n \"status\": true,"
<< "\n \"filepath\": \"/tmp/test_download_dir/" << test_file_name << "\"\n}";
string saved_response;
EXPECT_CALL(mock_msg, sendMessage(
false,
_,
I_Messaging::Method::POST,
"127.0.0.1",
response_port,
_,
"/show-download-status",
_,
_,
_
)).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe<string>(string()))));
stringstream is;
is << "{\"file_name\": \"" << test_file_name << "\","
<< "\"response_port\": " << response_port << ","
<< "\"url\": \"" << url << "\","
<< "\"port\": " << port << ","
<< "\"uuid\": \"" << uuid << "\"}";
rest_handler->performRestCall(is);
downloading_routine();
EXPECT_THAT(saved_response, HasSubstr(expected_response.str()));
EXPECT_THAT(capture_debug.str(), HasSubstr(success_msg));
}
unique_ptr<ServerRest> rest_handler;
ostringstream capture_debug;
I_MainLoop::Routine downloading_routine;
MessagingDownloaderServer messaging_downloader;
NiceMock<MockTimeGet> mock_time;
StrictMock<MockAgentDetails> mock_agent_details;
StrictMock<MockMessaging> mock_msg;
StrictMock<MockRestApi> rest;
StrictMock<MockMainLoop> mock_ml;
::Environment env;
ConfigComponent conf;
};
TEST_F(MessagingDownloaderServerTest, do_nothing)
{
}
TEST_F(MessagingDownloaderServerTest, add_one_secured_request)
{
string test_file_name = "test_file_name";
string host = "test_host";
string url = "https://test_host/test_url";
string uuid = "111";
string success_msg = "Successfully downloaded the file. File name: " + test_file_name;
unsigned int port = 443;
unsigned int response_port = 123;
expectRequestSuccess(test_file_name, host, url, uuid, port, response_port, success_msg);
}
TEST_F(MessagingDownloaderServerTest, add_one_non_secured_request)
{
string test_file_name = "test_file_name";
string host = "test_host";
string url = "http://test_host/test_url";
string uuid = "111";
string success_msg = "Successfully downloaded the file. File name: " + test_file_name;
unsigned int port = 80;
unsigned int response_port = 123;
expectRequestSuccess(test_file_name, host, url, uuid, port, response_port, success_msg);
}
TEST_F(MessagingDownloaderServerTest, add_multiple_requests)
{
string test_file_name1 = "test_file_name1";
string test_file_name2 = "test_file_name2";
string host = "test_host";
string url = "https://test_host/test_url";
string uuid = "111";
string success_msg1 = "Successfully downloaded the file. File name: " + test_file_name1;
string success_msg2 = "Successfully downloaded the file. File name: " + test_file_name2;
unsigned int port = 443;
unsigned int response_port = 123;
expectRequestSuccess(test_file_name1, host, url, uuid, port, response_port, success_msg1);
expectRequestSuccess(test_file_name2, host, url, uuid, port, response_port, success_msg2);
}
TEST_F(MessagingDownloaderServerTest, add_same_request_twice)
{
string test_file_name = "test_file_name";
string host = "test_host";
string url = "https://test_host/test_url";
string uuid = "111";
unsigned int response_port = 123;
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false)
).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0)));
EXPECT_CALL(
mock_msg,
sendMessage(true, "", I_Messaging::Method::GET, host, 442, _, url, _, _, _)
).WillOnce(Return(Maybe<string>(string("test_body"))));
stringstream expected_response;
expected_response
<< "\n \"status\": true,"
<< "\n \"filepath\": \"/tmp/test_download_dir/" << test_file_name << "\"\n}";
string saved_response;
EXPECT_CALL(mock_msg, sendMessage(
false,
_,
I_Messaging::Method::POST,
"127.0.0.1",
response_port,
_,
"/show-download-status",
_,
_,
_
)).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe<string>(string()))));
stringstream is;
is
<< "{\"file_name\": \"" << test_file_name << "\","
<< "\"response_port\": " << response_port << ","
<< "\"uuid\": \"" << uuid << "\","
<< "\"port\": 442,"
<< "\"url\": \"" << url << "\"}";
rest_handler->performRestCall(is);
rest_handler->doCall();
downloading_routine();
EXPECT_THAT(saved_response, HasSubstr(expected_response.str()));
EXPECT_THAT(
capture_debug.str(),
HasSubstr("Failed to download the file. Similar download request already exists.")
);
}
TEST_F(MessagingDownloaderServerTest, add_request_that_fails)
{
string test_file_name = "test_file_name";
string host = "test_host";
string url = "https://test_host/test_url";
string uuid = "111";
unsigned int response_port = 123;
unsigned int additional_port_test = 123;
Maybe<string> err = genError("no");
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, _, false)
).WillOnce(DoAll(SaveArg<1>(&downloading_routine), Return(0)));
EXPECT_CALL(
mock_msg,
sendMessage(true, "", I_Messaging::Method::GET, host, additional_port_test, _, url, _, _, _)
).WillOnce(Return(err));
stringstream expected_response;
expected_response
<< "\n \"status\": false,"
<< "\n \"error\": \"Failed during the downloading process.\"\n}";
string saved_response;
EXPECT_CALL(mock_msg, sendMessage(
false,
_,
I_Messaging::Method::POST,
"127.0.0.1",
response_port,
_,
"/show-download-status",
_,
_,
_
)).WillOnce(DoAll(SaveArg<1>(&saved_response), Return(Maybe<string>(string()))));
stringstream is;
is
<< "{\"file_name\": \"" << test_file_name << "\","
<< "\"response_port\": " << response_port << ","
<< "\"url\": \"" << url << "\","
<< "\"port\": " << additional_port_test << ","
<< "\"uuid\": \"" << uuid << "\"}";
rest_handler->performRestCall(is);
downloading_routine();
EXPECT_THAT(saved_response, HasSubstr(expected_response.str()));
EXPECT_THAT(capture_debug.str(), HasSubstr("Failed to download file. File name: test_file_name"));
}
TEST_F(MessagingDownloaderServerTest, download_with_same_filename)
{
string test_file_name = "test_file_name";
string host = "test_host";
string url1 = "https://test_host/test_url1";
string url2 = "https://test_host/test_url2";
string uuid = "111";
unsigned int port = 443;
string success_msg = "Successfully downloaded the file. File name: " + test_file_name;
unsigned int response_port = 123;
expectRequestSuccess(test_file_name, host, url1, uuid, port, response_port, success_msg);
stringstream is;
is
<< "{\"file_name\": \"" << test_file_name << "\","
<< "\"response_port\": " << response_port << ","
<< "\"port\": " << port << ","
<< "\"url\": \"" << url2 << "\"}";
rest_handler->performRestCall(is);
EXPECT_THAT(
capture_debug.str(),
HasSubstr("The file with the name 'test_file_name' is already exist in the downloading directory")
);
}

3
components/security_apps/layer_7_access_control/layer_7_access_control.cc
View File

@ -278,7 +278,8 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen
<< genLogIPField("destinationIP", HttpTransactionData::listening_ip_ctx)
<< LogField("securityAction", security_action)
<< LogField("sourceIP", source_ip)
<< LogField("externalVendorName", "crowdsec")
<< LogField("externalVendorName", "CrowdSec")
<< LogField("waapIncidentType", "CrowdSec")
<< ip_reputation.getCrowdsecEventId()
<< ip_reputation.getType()
<< ip_reputation.getOrigin()

6
components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc
View File

@ -7,6 +7,7 @@
#include "mock/mock_http_manager.h"
#include "mock/mock_logging.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "intelligence_comp_v2.h"
#include "agent_details.h"
@ -27,6 +28,7 @@ public:
EXPECT_CALL(mock_time, getMonotonicTime()).WillRepeatedly(Return(chrono::seconds(60)));
EXPECT_CALL(mock_ml, doesRoutineExist(_)).WillRepeatedly(Return(true));
EXPECT_CALL(mock_ml, stop(_)).WillRepeatedly(Return());
EXPECT_CALL(mock_ml, addRecurringRoutine(_, _, _, "Sending intelligence invalidation", _));
env.preload();
env.init();
config.preload();
@ -57,6 +59,7 @@ public:
StrictMock<MockTimeGet> mock_time;
StrictMock<MockMainLoop> mock_ml;
StrictMock<MockMessaging> messaging_mock;
NiceMock<MockRestApi> mock_rest;
AgentDetails agent_details;
IntelligenceComponentV2 intelligence_comp;
Context ctx;
@ -243,7 +246,8 @@ Layer7AccessControlTest::verifyReport(
EXPECT_THAT(log, HasSubstr("\"httpMethod\": \"POST\""));
EXPECT_THAT(log, HasSubstr("\"ipProtocol\": \"http\""));
EXPECT_THAT(log, HasSubstr("\"destinationIP\": \"5.6.7.8\""));
EXPECT_THAT(log, HasSubstr("\"externalVendorName\": \"crowdsec\""));
EXPECT_THAT(log, HasSubstr("\"externalVendorName\": \"CrowdSec\""));
EXPECT_THAT(log, HasSubstr("\"waapIncidentType\": \"CrowdSec\""));
EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationId\": 2253734"));
EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendedAction\": \"ban\""));
EXPECT_THAT(log, HasSubstr("\"externalVendorRecommendationOrigin\": \"cscli\""));

16
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -70,6 +70,14 @@ checkHasSDWan(const string &command_output)
return genError("Current host does not have SDWAN capability");
}
Maybe<string>
checkCanUpdateSDWanData(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
return string("true");
}
Maybe<string>
getMgmtObjType(const string &command_output)
{
@ -157,6 +165,14 @@ getGWVersion(shared_ptr<istream> file_stream)
return getMgmtObjAttr(file_stream, "svn_version_name ");
}
Maybe<string>
checkIfSdwanRunning(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
return genError("Could not determine if sd-wan is running or not");
}
Maybe<string>
getSmbObjectName(const string &command_output)
{

5
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -31,6 +31,11 @@
#if defined(gaia) || defined(smb)
SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER("canUpdateSDWanData", "cpsdwan get_data | jq -r .can_update_sdwan_data", checkCanUpdateSDWanData)
SHELL_CMD_HANDLER(
"isSdwanRunning",
"[ -v $(pidof cp-nano-sdwan) ] && echo 'false' || echo 'true'",
checkIfSdwanRunning)
#endif //gaia || smb
#if defined(gaia)

1
components/security_apps/orchestration/include/get_status_rest.h
View File

@ -14,7 +14,6 @@
#ifndef __GET_STATUS_RES_H__
#define __GET_STATUS_RES_H__
#include "i_messaging_downloader.h"
#include "i_messaging.h"
#include "i_mainloop.h"
#include "i_shell_cmd.h"

39
components/security_apps/orchestration/include/mock/mock_messaging_downloader.h
View File

@ -1,39 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __MOCK_MESSAGING_DOWNLOADER_H__
#define __MOCK_MESSAGING_DOWNLOADER_H__
#include "cptest.h"
#include <string>
#include "i_messaging_downloader.h"
class MockMessagingDownloader
:
public Singleton::Provide<I_MessagingDownloader>::From<MockProvider<I_MessagingDownloader>>
{
public:
MOCK_METHOD4(
downloadFile,
bool(
const std::string &,
const std::string &,
OnCompleteCB,
const unsigned int
)
);
};
#endif // __MOCK_MESSAGING_DOWNLOADER_H__

4
components/security_apps/orchestration/include/mock/mock_service_controller.h
View File

@ -26,6 +26,10 @@ class MockServiceController :
public:
MOCK_METHOD0(refreshPendingServices, void());
MOCK_METHOD0(mockMoveChangedPolicies, std::set<std::string>());
std::set<std::string> tmp;
std::set<std::string> && moveChangedPolicies() override { tmp = mockMoveChangedPolicies(); return std::move(tmp); }
MOCK_METHOD0(doesFailedServicesExist, bool());
MOCK_METHOD0(clearFailedServices, void());

1
components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc
View File

@ -740,6 +740,7 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopyWithError)
orch_service_name;
EXPECT_CALL(mock_orchestration_tools, doesFileExist(path)).WillOnce(Return(false)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(path, path + backup_ext + temp_ext)).WillOnce(Return(false));
EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(hostname));
EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
}

49
components/security_apps/orchestration/manifest_controller/manifest_handler.cc
View File

@ -145,23 +145,21 @@ ManifestHandler::downloadPackages(
}
}
downloaded_packages.clear();
auto agent_details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'";
string install_error;
if (is_clean_installation) {
string error_hostname_addition = "";
auto maybe_hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
if (maybe_hostname.ok()) {
error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'";
}
install_error =
"Critical Error: Agent/Gateway was not fully deployed" +
error_hostname_addition +
"Critical Error: Agent/Gateway was not fully deployed " +
err_hostname +
" and is not enforcing a security policy. Retry installation or contact Check Point support.";
} else {
auto agent_details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
install_error =
"Warning: Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software. Contact Check Point support.";
"Warning: Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software. Contact Check Point support.";
}
auto orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<ManifestHandler>();
@ -221,11 +219,13 @@ ManifestHandler::installPackages(
orchestration_status->writeStatusToFile();
bool self_update_status = selfUpdate(package, current_packages, package_handler_path);
if (!self_update_status) {
auto agent_details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
auto details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + details->getAgentId()) + "'";
string install_error =
"Warning: Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software. Contact Check Point support.";
"Warning: Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software. Contact Check Point support.";
if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) {
orchestration_status->setFieldStatus(
OrchestrationStatusFieldType::MANIFEST,
@ -278,23 +278,20 @@ ManifestHandler::installPackages(
}
if (!current_result) {
auto agent_details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" +agent_details->getAgentId()) + "'";
string install_error;
if (is_clean_installation) {
string error_hostname_addition = "";
auto maybe_hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
if (maybe_hostname.ok()) {
error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'";
}
install_error =
"Critical Error: Agent/Gateway was not fully deployed" +
error_hostname_addition +
"Critical Error: Agent/Gateway was not fully deployed " +
err_hostname +
" and is not enforcing a security policy. Retry installation or contact Check Point support.";
} else {
auto agent_details = Singleton::Consume<I_AgentDetails>::by<ManifestHandler>();
install_error =
"Warning: Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software. Contact Check Point support.";
"Warning: Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software. Contact Check Point support.";
}
corrupted_packages.insert(make_pair(package_name, package));
dbgWarning(D_ORCHESTRATOR) << "Failed to install package. Package: " << package_name;

176
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -41,6 +41,7 @@
#include "tenant_profile_pair.h"
#include "env_details.h"
#include "hybrid_communication.h"
#include "agent_core_utilities.h"
using namespace std;
using namespace chrono;
@ -48,6 +49,14 @@ using namespace ReportIS;
USE_DEBUG_FLAG(D_ORCHESTRATOR);
static const string ls_prefix = "ls ";
static const string extract_tenant_profile_suffix =
"| grep tenant "
"| cut -d '_' -f 2,4 "
"| sort --unique "
"| awk -F '_' '{ printf \"%s %s \",$1,$2 }'";
class HealthCheckStatusListener : public Listener<HealthCheckStatusEvent>
{
public:
@ -193,6 +202,7 @@ public:
ReportIS::Audience::INTERNAL
);
hybrid_mode_metric.registerListener();
loadExistingTenantsFromConfDir();
}
void
@ -237,7 +247,9 @@ private:
}
policy_version = service_controller->getPolicyVersion();
Singleton::Consume<I_OrchestrationStatus>::by<OrchestrationComp>()->setPolicyVersion(policy_version);
if (!policy_version.empty()) {
Singleton::Consume<I_OrchestrationStatus>::by<OrchestrationComp>()->setPolicyVersion(policy_version);
}
} else {
dbgDebug(D_ORCHESTRATOR) << "Orchestration is running for the first time";
enforce_policy_flag = true;
@ -277,6 +289,59 @@ private:
return authentication_res;
}
void
loadExistingTenantsFromConfDir()
{
dbgTrace(D_ORCHESTRATOR) << "Load existing tenants and profiles from the configuration folder";
string global_conf_dir = getConfigurationWithDefault<string>(
getFilesystemPathConfig()+ "/conf/",
"orchestration",
"Conf dir"
);
string shell_cmd_string = ls_prefix + global_conf_dir + extract_tenant_profile_suffix;
auto shell = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
Maybe<string> output_res = shell->getExecOutput(shell_cmd_string);
if (!output_res.ok()) {
dbgWarning(D_ORCHESTRATOR)
<< "Failed to load existing tenants from configuration folder: " + output_res.getErr();
return;
}
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
stringstream ss(output_res.unpack());
string tenant_id;
string profile_id;
while (!ss.eof() && getline(ss, tenant_id, ' ') && !ss.eof() && getline(ss, profile_id, ' ')) {
dbgTrace(D_ORCHESTRATOR) << "Add existing tenant_" + tenant_id + "_profile_" + profile_id;
tenant_manager->addActiveTenantAndProfile(tenant_id, profile_id);
}
}
void
deleteInactiveTenantProfileFiles(const string &tenant_id, const string &profile_id)
{
string global_conf_dir = getConfigurationWithDefault<string>(
getFilesystemPathConfig()+ "/conf/",
"orchestration",
"Conf dir"
);
string tenant_and_profile_suffix = "tenant_" + tenant_id + "_profile_" + profile_id;
string virtual_policy_dir = global_conf_dir + tenant_and_profile_suffix;
dbgTrace(D_ORCHESTRATOR) << "Delete virtual policy folder : " << virtual_policy_dir;
if (!NGEN::Filesystem::deleteDirectory(virtual_policy_dir, true)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to delete virtual policy folder : " << virtual_policy_dir;
}
string settings_file_path = virtual_policy_dir + "_settings.json";
dbgTrace(D_ORCHESTRATOR) << "Delete settings file " << settings_file_path;
if (!NGEN::Filesystem::deleteFile(settings_file_path)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to delete virtual policy settings file : " << settings_file_path;
}
}
Maybe<OrchestrationPolicy>
loadOrchestrationPolicy()
{
@ -404,33 +469,30 @@ private:
dbgInfo(D_ORCHESTRATOR) << "There is a new manifest file.";
GetResourceFile resource_file(GetResourceFile::ResourceFileType::MANIFEST);
Maybe<string> new_manifest_file =
Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
orch_manifest.unpack(),
I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
resource_file
);
Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
orch_manifest.unpack(),
I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
resource_file
);
auto orch_status = Singleton::Consume<I_OrchestrationStatus>::by<OrchestrationComp>();
auto service_controller = Singleton::Consume<I_ServiceController>::by<OrchestrationComp>();
auto agent_details = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>();
static int service_to_port_size = service_controller->getServiceToPortMap().size();
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'";
if (!new_manifest_file.ok()) {
string install_error;
if (!service_to_port_size) {
string error_hostname_addition = "";
auto maybe_hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
if (maybe_hostname.ok()) {
error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'";
}
install_error =
"Critical Error: Agent/Gateway was not fully deployed" +
error_hostname_addition +
"Critical Error: Agent/Gateway was not fully deployed " +
err_hostname +
" and is not enforcing a security policy. Retry installation or contact Check Point support.";
} else {
install_error =
"Warning: Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software. Contact Check Point support.";
"Warning: Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software. Contact Check Point support.";
}
dbgTrace(D_ORCHESTRATOR)
<< "Manifest failed to be updated. Error: "
@ -455,9 +517,9 @@ private:
auto manifest_controller = Singleton::Consume<I_ManifestController>::by<OrchestrationComp>();
if (!manifest_controller->updateManifest(new_manifest_file.unpack())) {
string install_error =
"Warning: Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software. Contact Check Point support.";
"Warning: Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software. Contact Check Point support.";
string current_error = orch_status->getManifestError();
if (current_error.find("Gateway was not fully deployed") == string::npos) {
orch_status->setFieldStatus(
@ -499,9 +561,9 @@ private:
}
string manifest_success_notification_message(
"Agent/Gateway '" +
agent_details->getAgentId() +
"' software update succeeded. Agent is running latest software."
"Agent/Gateway " +
err_hostname +
" software update succeeded. Agent is running latest software."
);
LogGen manifest_success_notification(
manifest_success_notification_message,
@ -517,7 +579,6 @@ private:
return Maybe<void>();
}
// LCOV_EXCL_START Reason: future changes will be done
bool
updateServiceConfigurationFromBackup()
{
@ -565,7 +626,6 @@ private:
dbgWarning (D_ORCHESTRATOR) << "Failed to load Orchestration policy.";
return false;
}
// LCOV_EXCL_STOP
string
updatePolicyAndFogAddress(const OrchestrationPolicy &orchestration_policy)
@ -582,7 +642,9 @@ private:
auto service_controller = Singleton::Consume<I_ServiceController>::by<OrchestrationComp>();
string new_policy_version = service_controller->getPolicyVersion();
Singleton::Consume<I_OrchestrationStatus>::by<OrchestrationComp>()->setPolicyVersion(new_policy_version);
if (!new_policy_version.empty()) {
Singleton::Consume<I_OrchestrationStatus>::by<OrchestrationComp>()->setPolicyVersion(new_policy_version);
}
auto update_communication = Singleton::Consume<I_UpdateCommunication>::by<OrchestrationComp>();
auto path_policy_version = update_communication->sendPolicyVersion(new_policy_version);
if (!path_policy_version.ok()) {
@ -617,7 +679,7 @@ private:
"last fog policy file extension"
);
if (!orchestration_tools->copyFile(new_policy_file.unpack(), conf_path + last_ext)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext;
dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new policy file to " << conf_path + last_ext;
}
// Calculate the changes between the existing policy to the new one.
@ -648,6 +710,11 @@ private:
"Settings file path"
);
set<string> changed_policy_files = service_controller->moveChangedPolicies();
for (const string &changed_policy_file : changed_policy_files) {
orchestration_tools->writeFile("{}\n", changed_policy_file);
}
service_controller->updateServiceConfiguration(policy_file, setting_file, data_updates);
LogGen(
error_str,
@ -661,6 +728,7 @@ private:
return genError(error_str);
}
service_controller->moveChangedPolicies();
// Reload the orchestration policy, in case of the policy updated
auto orchestration_policy = loadDefaultOrchestrationPolicy();
@ -672,8 +740,6 @@ private:
if (new_policy_version.empty()) {
return genError("Failed to load Orchestration new policy file.");
}
reloadConfiguration();
if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.orchestration.reportAgentDetail")) {
service_controller->clearFailedServices();
reportAgentDetailsMetaData();
@ -1021,12 +1087,13 @@ private:
vector<string> data_updates;
update_results[OrchestrationStatusConfigType::DATA] = handleDataUpdate(orch_data, data_updates);
update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate(
orch_policy,
settings_path,
data_updates
);
if (!orch_manifest.ok() && orch_policy.ok()) {
update_results[OrchestrationStatusConfigType::POLICY] = handlePolicyUpdate(
orch_policy,
settings_path,
data_updates
);
}
if (!orch_policy.ok() && (data_updates.size() > 0 || settings_path != "")) {
auto service_controller = Singleton::Consume<I_ServiceController>::by<OrchestrationComp>();
bool res = service_controller->updateServiceConfiguration(
@ -1046,22 +1113,19 @@ private:
string recommended_fix;
string msg;
bool is_deploy_error = current_error.find("Critical") != string::npos;
auto hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
string err_hostname = (hostname.ok() ? "on host '" + *hostname : "'" + agent_details->getAgentId()) + "'";
if (is_deploy_error) {
string error_hostname_addition = "";
auto maybe_hostname = Singleton::Consume<I_DetailsResolver>::by<ManifestHandler>()->getHostname();
if (maybe_hostname.ok()) {
error_hostname_addition = " on host '" + maybe_hostname.unpack() + "'";
}
msg =
"Agent/Gateway was not fully deployed" +
error_hostname_addition +
"Agent/Gateway was not fully deployed " +
err_hostname +
" and is not enforcing a security policy.";
recommended_fix = "Retry installation or contact Check Point support.";
} else if (current_error.find("Warning") != string::npos) {
msg =
"Agent/Gateway '" +
agent_details->getAgentId() +
"' software update failed. Agent is running previous software.";
"Agent/Gateway " +
err_hostname +
" software update failed. Agent is running previous software.";
recommended_fix = "Contact Check Point support.";
}
if (!msg.empty() && !recommended_fix.empty()) {
@ -1114,7 +1178,11 @@ private:
bool is_empty = true;
GetResourceFile resource_v_policy_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY);
I_Downloader *downloader = Singleton::Consume<I_Downloader>::by<OrchestrationComp>();
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
map<string, set<string>> profiles_to_be_deleted =
tenant_manager->fetchAndUpdateActiveTenantsAndProfiles(false);
for (const auto &tenant: *updated_policy_tenants) {
profiles_to_be_deleted[tenant.getTenantID()].erase(tenant.getProfileID());
if (!tenant.getVersion().empty()) {
is_empty = false;
@ -1127,7 +1195,6 @@ private:
<< tenant.getTenantID()
<< " Profile: "
<< profile_to_use;
auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), profile_to_use);
resource_v_policy_file.addTenant(
@ -1204,6 +1271,22 @@ private:
}
}
for (const auto &tenant_profile_set : profiles_to_be_deleted) {
auto tenant_id = tenant_profile_set.first;
for (const auto &profile_id: tenant_profile_set.second) {
dbgTrace(D_ORCHESTRATOR)
<< "Delete configuration files for inactive profile: "
<< "Tenant ID: "
<< tenant_id
<< ", Profile ID: "
<< profile_id;
tenant_manager->deactivateTenant(tenant_id, profile_id);
deleteInactiveTenantProfileFiles(tenant_id, profile_id);
}
}
clearOldTenants();
for (auto it = sorted_files.begin(); it != sorted_files.end(); it++) {
const auto &downloaded_files = *it;
auto files = downloaded_files.second;
@ -1574,6 +1657,9 @@ private:
auto server = TagAndEnumManagement::convertStringToTag(server_name);
if (server.ok()) tags.insert(*server);
if (getAttribute("no-setting", "CROWDSEC_ENABLED") == "true") tags.insert(Tags::CROWDSEC);
if (getAttribute("no-setting", "PLAYGROUND") == "true") tags.insert(Tags::PLAYGROUND);
Report registration_report(
"Local Agent Data",
Singleton::Consume<I_TimeGet>::by<OrchestrationComp>()->getWalltime(),

29
components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc
View File

@ -17,7 +17,6 @@
#include "mock/mock_time_get.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_tenant_manager.h"
#include "mock/mock_messaging_downloader.h"
#include "config.h"
#include "config_component.h"
#include "agent_details.h"
@ -35,7 +34,6 @@ public:
mockRestCall(RestAction::SET, "new-configuration", _)
).WillOnce(WithArg<2>(Invoke(this, &OrchestrationMultitenancyTest::setNewConfiguration)));
EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::microseconds(0)));
EXPECT_CALL(
mock_ml,
addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _)
@ -76,6 +74,17 @@ public:
).WillOnce(Return(true));
doEncrypt();
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"ls /etc/cp/conf/"
"| grep tenant "
"| cut -d '_' -f 2,4 "
"| sort --unique "
"| awk -F '_' '{ printf \"%s %s \",$1,$2 }'",
_,
_
)).WillOnce(Return(Maybe<string>(string(""))));
orchestration_comp.init();
}
@ -173,7 +182,6 @@ public:
StrictMock<MockServiceController> mock_service_controller;
StrictMock<MockManifestController> mock_manifest_controller;
StrictMock<MockUpdateCommunication> mock_update_communication;
StrictMock<MockMessagingDownloader> mock_messaging_downloader;
StrictMock<MockTenantManager> tenant_manager;
NiceMock<MockOrchestrationStatus> mock_status;
@ -225,6 +233,10 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
string first_policy_version = "";
string host_url = "https://" + host_address + "/";
stringstream debug_output;
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::TRACE);
EXPECT_CALL(
rest,
mockRestCall(RestAction::ADD, "proxy", _)
@ -266,7 +278,13 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
set<string> active_tenants = { "1236", "1235" };
map<string, set<string>> old_tenant_profile_set;
set<string> old_profiles = { "123123" };
old_tenant_profile_set["321321"] = old_profiles;
EXPECT_CALL(tenant_manager, fetchActiveTenants()).WillOnce(Return(active_tenants));
EXPECT_CALL(tenant_manager, fetchAndUpdateActiveTenantsAndProfiles(false))
.WillOnce(Return(old_tenant_profile_set));
EXPECT_CALL(tenant_manager, deactivateTenant("321321", "123123")).Times(1);
EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1235", "2311"));
EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1236", "2611"));
@ -479,4 +497,9 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
try {
runRoutine();
} catch (const invalid_argument& e) {}
string debug_str_folder = "Delete virtual policy folder : /etc/cp/conf/tenant_321321_profile_123123";
string debug_str_settings = "Delete settings file /etc/cp/conf/tenant_321321_profile_123123_settings.json";
EXPECT_THAT(debug_output.str(), HasSubstr(debug_str_folder));
EXPECT_THAT(debug_output.str(), HasSubstr(debug_str_settings));
Debug::setNewDefaultStdout(&cout);
}

255
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@ -16,7 +16,6 @@
#include "mock/mock_messaging.h"
#include "mock/mock_time_get.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_messaging_downloader.h"
#include "mock/mock_tenant_manager.h"
#include "config.h"
#include "config_component.h"
@ -95,6 +94,17 @@ public:
)).WillRepeatedly(DoAll(SaveArg<1>(&message_body), Return(Maybe<string>(string("")))));
doEncrypt();
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"ls /etc/cp/conf/"
"| grep tenant "
"| cut -d '_' -f 2,4 "
"| sort --unique "
"| awk -F '_' '{ printf \"%s %s \",$1,$2 }'",
_,
_
)).WillOnce(Return(Maybe<string>(string(""))));
orchestration_comp.init();
}
@ -138,6 +148,7 @@ public:
EXPECT_CALL(mock_details_resolver, isVersionEqualOrAboveR8110()).WillRepeatedly(Return(false));
EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx));
EXPECT_CALL(mock_details_resolver, getAgentVersion()).WillRepeatedly(Return("1.1.1"));
EXPECT_CALL(mock_details_resolver, getHostname()).WillRepeatedly(Return(string("hostname")));
map<string, string> resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}});
EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details));
@ -277,7 +288,6 @@ public:
unique_ptr<ServerRest> rest_status;
StrictMock<MockOrchestrationTools> mock_orchestration_tools;
StrictMock<MockDownloader> mock_downloader;
StrictMock<MockMessagingDownloader> mock_messaging_downloader;
StrictMock<MockShellCmd> mock_shell_cmd;
StrictMock<MockMessaging> mock_message;
StrictMock<MockRestApi> rest;
@ -538,6 +548,222 @@ TEST_F(OrchestrationTest, check_sending_registration_data)
EXPECT_THAT(message_body, HasSubstr("\"NGINX Server\""));
}
TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback)
{
Debug::setUnitTestFlag(D_CONFIG, Debug::DebugLevel::TRACE);
waitForRestCall();
preload();
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(I_MainLoop::RoutineType::Offline, _, "Send policy update report", _)
).WillOnce(Return(1));
EXPECT_CALL(
rest,
mockRestCall(RestAction::ADD, "proxy", _)
).WillOnce(WithArg<2>(Invoke(this, &OrchestrationTest::restHandler)));
EXPECT_CALL(mock_status, setFogAddress(host_url)).Times(2);
string config_json =
"{\n"
"\"agentSettings\": [{\n"
"\"key\": \"agent.config.orchestration.reportAgentDetail\",\n"
"\"id\": \"id1\",\n"
"\"value\": \"true\"\n"
"}]\n"
"}\n";
istringstream ss(config_json);
Singleton::Consume<Config::I_Config>::from(config_comp)->loadConfiguration(ss);
init();
// All duplicates should be removed in INXT-35947
string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy";
string manifest_file_path = "/etc/cp/conf/manifest.json";
string setting_file_path = "/etc/cp/conf/settings.json";
string policy_file_path = "/etc/cp/conf/policy.json";
string policy_file_path_bk = "/etc/cp/conf/policy.json.bk";
string last_policy_file_path = "/etc/cp/conf/policy.json.last";
string data_file_path = "/etc/cp/conf/data.json";
string host_address = "1.2.3.5";
string new_host_address = "6.2.3.5";
string new_host_url = "https://" + new_host_address + "/test/";
string new_policy_path = "/some-path";
string manifest_checksum = "manifest";
string policy_checksum = "policy";
string settings_checksum = "settings";
string data_checksum = "data";
string new_policy_checksum= "111111";
string second_val = "12";
string third_val = "13";
Maybe<string> policy_response(
string(
"{\n"
" \"fog-address\": \"" + host_url + "\",\n"
" \"agent-type\": \"test\",\n"
" \"pulling-interval\": 25,\n"
" \"error-pulling-interval\": 15\n"
"}"
)
);
Maybe<string> new_policy_response(
string(
"{\n"
" \"fog-address\": \"" + new_host_url + "\",\n"
" \"agent-type\": \"test\",\n"
" \"pulling-interval\": 25,\n"
" \"error-pulling-interval\": 15\n"
"}"
)
);
set<string> expected_changed_policies = {};
EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies));
EXPECT_CALL(mock_status, setFogAddress(new_host_url));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
// Rollback related test: The readFile function is called 3 times:
// 1. Read the current policy file
// 2. Read the new policy file - The one that should fail
// 3. Read the current policy file again - The one that should be restored
EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path))
.WillOnce(Return(policy_response))
.WillOnce(Return(new_policy_response))
.WillOnce(Return(policy_response));
EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last"))
.WillOnce(Return(true));
EXPECT_CALL(mock_message, setActiveFog(host_address, 443, true, MessageTypeTag::GENERIC))
.Times(2).WillRepeatedly(Return(true));
EXPECT_CALL(mock_update_communication, setAddressExtenesion("")).Times(2);
EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe<void>()));
expectDetailsResolver();
EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path))
.WillOnce(Return(manifest_checksum));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, setting_file_path))
.WillOnce(Return(settings_checksum));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, policy_file_path))
.WillOnce(Return(policy_checksum));
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, data_file_path))
.WillOnce(Return(data_checksum));
// Rollback related test: After failing to update the policy file, the policy version should be restored
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(5)
.WillOnce(ReturnRef(first_policy_version))
.WillOnce(ReturnRef(first_policy_version))
.WillOnce(ReturnRef(second_val))
.WillOnce(ReturnRef(third_val))
.WillOnce(ReturnRef(second_val)
);
EXPECT_CALL(mock_status, setPolicyVersion(third_val));
EXPECT_CALL(mock_status, setPolicyVersion(second_val));
EXPECT_CALL(mock_update_communication, sendPolicyVersion("13")).Times(1).WillOnce(Return(Maybe<void>()));
// Rollback related test: The old policy version 12 is restored
EXPECT_CALL(mock_update_communication, sendPolicyVersion("12")).Times(1).WillOnce(Return(Maybe<void>()));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
[&](CheckUpdateRequest &req)
{
EXPECT_THAT(req.getPolicy(), IsValue(policy_checksum));
EXPECT_THAT(req.getSettings(), IsValue(settings_checksum));
EXPECT_THAT(req.getManifest(), IsValue(manifest_checksum));
EXPECT_THAT(req.getData(), IsValue(data_checksum));
req = CheckUpdateRequest("", new_policy_checksum, "", "", "", "");
return Maybe<void>();
}
)
);
GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY);
EXPECT_CALL(
mock_downloader,
downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file)
).WillOnce(Return(Maybe<std::string>(new_policy_path)));
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _)
).WillOnce(Return(true));
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(new_policy_path, "", expected_data_types, "", "", _)
).WillOnce(Return(true));
EXPECT_CALL(
mock_message,
setActiveFog(new_host_address, 443, true, MessageTypeTag::GENERIC)
).WillOnce(Return(true));
EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test"));
EXPECT_CALL(mock_status, setLastUpdateAttempt());
EXPECT_CALL(
mock_status,
setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
);
EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
).WillOnce(
Invoke(
[](EnumArray<OrchestrationStatusConfigType, bool> arr)
{
EXPECT_EQ(arr[OrchestrationStatusConfigType::MANIFEST], false);
EXPECT_EQ(arr[OrchestrationStatusConfigType::POLICY], true);
EXPECT_EQ(arr[OrchestrationStatusConfigType::SETTINGS], false);
}
)
);
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
.WillOnce(
Invoke(
[] (chrono::microseconds microseconds)
{
EXPECT_EQ(1000000, microseconds.count());
}
)
)
.WillOnce(
Invoke(
[] (chrono::microseconds microseconds)
{
EXPECT_EQ(25000000, microseconds.count());
throw invalid_argument("stop while loop");
}
)
);
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(_, _, _)
).WillRepeatedly(Return(string("daniel\n1\n")));
EXPECT_CALL(mock_service_controller, clearFailedServices());
EXPECT_CALL(mock_service_controller, doesFailedServicesExist()).WillOnce(Return(true));
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path_bk, _, _, _, _, _)
).WillOnce(Return(true));
EXPECT_CALL(
mock_orchestration_tools,
copyFile(policy_file_path_bk, policy_file_path)
).WillOnce(Return(true));
try {
runRoutine();
} catch (const invalid_argument& e) {}
}
TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
{
waitForRestCall();
@ -597,6 +823,9 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
)
);
set<string> expected_changed_policies = {};
EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies));
EXPECT_CALL(mock_status, setFogAddress(new_host_url));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path))
@ -628,7 +857,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
.WillOnce(ReturnRef(second_val))
.WillOnce(ReturnRef(third_val)
);
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(third_val));
EXPECT_CALL(mock_update_communication, sendPolicyVersion("13")).Times(1).WillOnce(Return(Maybe<void>()));
@ -781,7 +1009,6 @@ TEST_F(OrchestrationTest, startOrchestrationPoliceWithFailures)
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
@ -910,7 +1137,6 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup)
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
[&](CheckUpdateRequest &req)
@ -1039,7 +1265,6 @@ TEST_F(OrchestrationTest, manifestUpdate)
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
[&](CheckUpdateRequest &req)
@ -1165,11 +1390,14 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
EXPECT_CALL(mock_status, setFogAddress(host_url));
vector<string> expected_data_types = {};
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(policy_file_path, setting_file_path, expected_data_types, "", "", _)
).Times(2).WillRepeatedly(Return(true));
set<string> expected_changed_policies = {};
EXPECT_CALL(mock_service_controller, mockMoveChangedPolicies()).WillOnce(Return(expected_changed_policies));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(orchestration_policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(orchestration_policy_file_path)).WillOnce(Return(response));
EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_path, policy_file_path + ".last"))
@ -1215,7 +1443,6 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
.WillOnce(ReturnRef(first_policy_version))
.WillOnce(ReturnRef(second_val)
);
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_status, setLastUpdateAttempt());
EXPECT_CALL(
mock_status,
@ -1250,8 +1477,8 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
EXPECT_CALL(
mock_service_controller,
updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "", _)).WillOnce(Return(false)
);
updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "", _)
).WillOnce(Return(false));
EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
.WillOnce(
@ -1343,7 +1570,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
[&](CheckUpdateRequest &req)
@ -1374,7 +1600,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
manifest_err
)
).Times(1);
EXPECT_CALL(mock_details_resolver, getHostname()).Times(2).WillRepeatedly(Return(string("hostname")));
EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(manifest_err));
EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
@ -1400,13 +1625,6 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
manifest_file
)
).WillOnce(Return(download_error));
EXPECT_CALL(mock_downloader,
downloadFileFromFog(
string("policy-checksum"),
Package::ChecksumTypes::SHA256,
policy_file
)
).WillOnce(Return(download_error));
EXPECT_CALL(mock_downloader,
downloadFileFromFog(
string("settings-checksum"),
@ -1754,7 +1972,6 @@ TEST_F(OrchestrationTest, dataUpdate)
EXPECT_CALL(mock_service_controller, getPolicyVersion())
.Times(2).WillRepeatedly(ReturnRef(first_policy_version));
EXPECT_CALL(mock_status, setPolicyVersion(first_policy_version));
EXPECT_CALL(mock_update_communication, getUpdate(_)).WillOnce(
Invoke(
[&](CheckUpdateRequest &req)

89
components/security_apps/orchestration/service_controller/service_controller.cc
View File

@ -181,6 +181,11 @@ ServiceDetails::serialize(Archive &ar)
ReconfStatus
ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy_version)
{
if(!isServiceActive()) {
dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " is inactive";
return ReconfStatus::INACTIVE;
}
SendConfigurations new_config(configuration_id, policy_version);
I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<ServiceController>();
@ -194,14 +199,12 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy
conn_flags,
"/set-new-configuration"
);
if (!res) {
if(!isServiceActive()) {
dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " is inactive";
return ReconfStatus::INACTIVE;
}
dbgDebug(D_ORCHESTRATOR) << "Service " << service_name << " didn't respond to new configuration request";
return ReconfStatus::FAILED;
}
auto service_details = Singleton::Consume<I_ServiceController>::by<ServiceDetails>();
if (new_config.finished.get()) {
@ -300,6 +303,8 @@ public:
void clearFailedServices() override;
set<string> && moveChangedPolicies() override;
private:
void cleanUpVirtualFiles();
@ -320,6 +325,8 @@ private:
void loadRegisteredServicesFromFile();
void writeRegisteredServicesToFile();
bool backupConfigurationFile(const string &configuration_file_path);
int configuration_id = 0;
map<string, ServiceDetails> registered_services;
map<string, ServiceDetails> pending_services;
@ -332,6 +339,10 @@ private:
map<int, string> services_reconf_ids;
string filesystem_prefix;
bool is_multi_tenant_env = false;
set<string> changed_policy_files;
I_OrchestrationTools *orchestration_tools = nullptr;
I_MainLoop *mainloop = nullptr;
};
class GetServicesPorts : public ServerRest
@ -410,6 +421,12 @@ ServiceController::Impl::doesFailedServicesExist()
}
// LCOV_EXCL_STOP
set<string> &&
ServiceController::Impl::moveChangedPolicies()
{
return move(changed_policy_files);
}
void
ServiceController::Impl::init()
{
@ -418,6 +435,9 @@ ServiceController::Impl::init()
rest->addRestCall<GetServicesPorts>(RestAction::SHOW, "all-service-ports");
rest->addRestCall<ServiceReconfStatusMonitor>(RestAction::SET, "reconf-status");
orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ServiceController>();
mainloop = Singleton::Consume<I_MainLoop>::by<ServiceController>();
Singleton::Consume<I_MainLoop>::by<ServiceController>()->addRecurringRoutine(
I_MainLoop::RoutineType::System,
chrono::seconds(
@ -613,6 +633,29 @@ ServiceController::Impl::refreshPendingServices()
writeRegisteredServicesToFile();
}
bool
ServiceController::Impl::backupConfigurationFile(const string &config_file_path)
{
uint max_backup_attempts = 3;
string backup_ext = getConfigurationWithDefault<string>(".bk", "orchestration", "Backup file extension");
string backup_file = config_file_path + backup_ext;
if (!orchestration_tools->doesFileExist(config_file_path)) {
dbgTrace(D_ORCHESTRATOR) << "File does not exist. File: " << config_file_path;
return true;
}
for (size_t i = 0; i < max_backup_attempts; i++) {
if (orchestration_tools->copyFile(config_file_path, backup_file)) {
return true;
}
mainloop->yield(false);
}
dbgWarning(D_ORCHESTRATOR) << "Failed to back up the file. File: " << config_file_path;
return false;
}
bool
ServiceController::Impl::updateServiceConfiguration(
const string &new_policy_path,
@ -665,19 +708,23 @@ ServiceController::Impl::updateServiceConfiguration(
return sendSignalForServices(nano_services_to_update, "");
}
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ServiceController>();
Maybe<string> loaded_json = orchestration_tools->readFile(new_policy_path);
if (!loaded_json.ok()) {
Maybe<string> loaded_policy_json = orchestration_tools->readFile(new_policy_path);
if (!loaded_policy_json.ok()) {
dbgWarning(D_ORCHESTRATOR)
<< "Failed to load new file: "
<< new_policy_path
<< ". Error: "
<< loaded_json.getErr();
<< loaded_policy_json.getErr();
return false;
}
auto all_security_policies = orchestration_tools->jsonObjectSplitter(loaded_json.unpack(), tenant_id, profile_id);
auto all_security_policies = orchestration_tools->jsonObjectSplitter(
loaded_policy_json.unpack(),
tenant_id,
profile_id
);
if (!all_security_policies.ok()) {
dbgWarning(D_ORCHESTRATOR)
@ -740,6 +787,7 @@ ServiceController::Impl::updateServiceConfiguration(
was_policy_updated = false;
continue;
}
changed_policy_files.insert(policy_file_path);
dbgInfo(D_ORCHESTRATOR) << "Successfully updated policy file. Policy name: " << single_policy.first;
@ -779,7 +827,7 @@ ServiceController::Impl::updateServiceConfiguration(
true :
sendSignalForServices(nano_services_to_update, version_value);
dbgTrace(D_ORCHESTRATOR) << "was_policy_updated: " << (was_policy_updated ? "true" : "false");
dbgTrace(D_ORCHESTRATOR) << "was policy updated: " << (was_policy_updated ? "true" : "false");
if (was_policy_updated) {
string config_file_path;
@ -798,24 +846,8 @@ ServiceController::Impl::updateServiceConfiguration(
return true;
}
string backup_ext = getConfigurationWithDefault<string>(".bk", "orchestration", "Backup file extension");
// Backup the current configuration file.
uint max_backup_attempts = 3;
bool is_backup_succeed = false;
string backup_file = config_file_path + backup_ext;
I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<ServiceController>();
for (size_t i = 0; i < max_backup_attempts; i++) {
if (orchestration_tools->copyFile(config_file_path, backup_file)) {
is_backup_succeed = true;
break;
}
mainloop->yield(false);
}
if (!is_backup_succeed) {
dbgWarning(D_ORCHESTRATOR) << "Failed to back up the policy file.";
if (!backupConfigurationFile(config_file_path)) {
dbgWarning(D_ORCHESTRATOR) << "Failed to backup the policy file.";
return false;
}
@ -921,7 +953,6 @@ ServiceController::Impl::updateServiceConfigurationFile(
dbgFlow(D_ORCHESTRATOR) << "Updating configuration. Config Name: " << configuration_name;
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ServiceController>();
if (orchestration_tools->doesFileExist(configuration_file_path)) {
Maybe<string> old_configuration = orchestration_tools->readFile(configuration_file_path);
if (old_configuration.ok()) {

199
components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc
View File

@ -42,9 +42,7 @@ public:
EXPECT_CALL(mock_rest_api, mockRestCall(RestAction::SET, "new-configuration", _)).WillOnce(
WithArg<2>(Invoke(this, &ServiceControllerTest::setNanoServiceConfig))
);
EXPECT_CALL(mock_ml, addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _)).WillOnce(Return(2));
EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::seconds(1)));
EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false)).WillOnce(Return(1));
config.init();
@ -257,6 +255,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
string general_settings_path = "/my/settings/path";
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
@ -349,6 +348,29 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control"
" --family family1 --id id2",
_,
_
)
).Times(4).WillRepeatedly(
InvokeWithoutArgs(
[&]() -> Maybe<string>
{
static int counter = 0;
if (counter++ < 2) {
return genError("Reached timeout while executing shell command:");
}
return string("registered and running");
}
)
);
string general_settings_path = "/my/settings/path";
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
@ -371,28 +393,6 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
)
).WillOnce(Return(Maybe<string>(reply_msg)));
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control"
" --family family1 --id id2",
_,
_
)
).Times(3).WillRepeatedly(
InvokeWithoutArgs(
[&]() -> Maybe<string>
{
static int counter = 0;
if (counter++ < 2) {
return genError("Reached timeout while executing shell command:");
}
return string("registered and running");
}
)
);
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path));
EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value);
@ -468,6 +468,7 @@ TEST_F(ServiceControllerTest, writeRegisteredServicesFromFile)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
string general_settings_path = "/my/settings/path";
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
@ -529,9 +530,7 @@ TEST_F(ServiceControllerTest, readRegisteredServicesFromFile)
EXPECT_CALL(mock_rest_api, mockRestCall(RestAction::SET, "new-configuration", _)).WillOnce(
WithArg<2>(Invoke(this, &ServiceControllerTest::setNanoServiceConfig))
);
EXPECT_CALL(mock_ml, addRecurringRoutine(I_MainLoop::RoutineType::System, _, _, _, _)).WillOnce(Return(2));
EXPECT_CALL(tenant_manager, getTimeoutVal()).WillOnce(Return(chrono::seconds(1)));
EXPECT_CALL(mock_ml, addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false)).WillOnce(Return(1));
config.init();
@ -609,6 +608,7 @@ TEST_F(ServiceControllerTest, noPolicyUpdate)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_status,
setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
@ -700,6 +700,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
@ -742,6 +743,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(l4_firewall));
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
@ -852,6 +854,122 @@ TEST_F(ServiceControllerTest, backup)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
getExecOutput(
"/etc/cp/watchdog/cp-nano-watchdog --status --verbose --service mock access control"
" --family family1 --id id2",
_,
_
)
).WillRepeatedly(Return(string("registered and running")));
string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
EXPECT_CALL(
mock_message,
sendMessage(
_,
_,
_,
"127.0.0.1",
l4_firewall_service_port,
_,
"/set-new-configuration",
_,
_,
MessageTypeTag::GENERIC
)
).WillOnce(Return(Maybe<string>(reply_msg)));
EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, ""));
EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
}
TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
{
string new_configuration = "{"
" \"version\": \"" + version_value + "\""
" \"l4_firewall\":"
" {"
" \"app\": \"netfilter\","
" \"l4_firewall_rules\": ["
" {"
" \"name\": \"allow_statefull_conns\","
" \"flags\": [\"established\"],"
" \"action\": \"accept\""
" },"
" {"
" \"name\": \"icmp drop\","
" \"flags\": [\"log\"],"
" \"services\": [{\"name\":\"icmp\"}],"
" \"action\": \"drop\""
" }"
" ]"
" }"
"}";
string l4_firewall = "{"
" \"app\": \"netfilter\","
" \"l4_firewall_rules\": ["
" {"
" \"name\": \"allow_statefull_conns\","
" \"flags\": [\"established\"],"
" \"action\": \"accept\""
" },"
" {"
" \"name\": \"icmp drop\","
" \"flags\": [\"log\"],"
" \"services\": [{\"name\":\"icmp\"}],"
" \"action\": \"drop\""
" }"
" ]"
"}";
string old_configuration = "{"
" \"version\": \"" + old_version + "\""
" \"app\": \"netfilter\","
" \"l4_firewall_rules\": ["
" {"
" \"name\": \"allow_statefull_conns\","
" \"flags\": [\"established\"],"
" \"action\": \"reject\""
" },"
" {"
" \"name\": \"icmp drop\","
" \"flags\": [\"log\"],"
" \"services\": [{\"name\":\"icmp\"}],"
" \"action\": \"drop\""
" }"
" ]"
"}";
Maybe<map<string, string>> json_parser_return =
map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
.WillOnce(Return(json_parser_return));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration));
EXPECT_CALL(mock_orchestration_status,
setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
EXPECT_CALL(
mock_orchestration_tools,
copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
).WillOnce(Return(true));
EXPECT_CALL(
mock_orchestration_tools,
writeFile(l4_firewall, l4_firewall_policy_path)).WillOnce(Return(true)
);
// backup file doesn't exist so the copyFile function should be called 0 times
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(false));
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension)).Times(0);
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
@ -997,6 +1115,7 @@ TEST_F(ServiceControllerTest, backupAttempts)
EXPECT_CALL(mock_ml, yield(false)).Times(2);
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, ""));
EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
@ -1081,6 +1200,7 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
@ -1112,6 +1232,11 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
).WillOnce(Return(Maybe<string>(reply_msg)));
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, ""));
set<string> changed_policies = {
"/etc/cp/conf/l4_firewall/l4_firewall.policy",
"/etc/cp/conf/orchestration/orchestration.policy"
};
EXPECT_EQ(i_service_controller->moveChangedPolicies(), changed_policies);
}
class TestSendRequestToService : public ClientRest
@ -1139,6 +1264,7 @@ TEST_F(ServiceControllerTest, emptyServices)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, ""));
}
@ -1326,24 +1452,6 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN);
EXPECT_CALL(
mock_message,
sendMessage(
true,
"{\n \"id\": 1,\n \"policy_version\": \"1.0.2\"\n}",
I_Messaging::Method::POST,
string("127.0.0.1"),
l4_firewall_service_port,
conn_flags,
string("/set-new-configuration"),
string(),
_,
MessageTypeTag::GENERIC
)
).WillOnce(Return(Maybe<string>(genError(""))));
EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2"));
EXPECT_CALL(
@ -1358,6 +1466,7 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, ""));
EXPECT_THAT(
@ -1568,6 +1677,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_file_path, new_policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(
mock_shell_cmd,
@ -1665,6 +1775,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
EXPECT_CALL(mock_ml, yield(false)).Times(AnyNumber());
EXPECT_CALL(

6
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

@ -208,12 +208,14 @@ void ConfidenceCalculator::pullProcessedData(const std::vector<std::string>& fil
{
dbgTrace(D_WAAP) << "Fetching the confidence set object";
bool is_first_pull = true;
bool is_ok = false;
for (auto file : files)
{
ConfidenceFileDecryptor getConfFile;
bool res = sendObjectWithRetry(getConfFile,
I_Messaging::Method::GET,
getUri() + "/" + file);
is_ok |= res;
if (res && getConfFile.getConfidenceSet().ok())
{
mergeFromRemote(getConfFile.getConfidenceSet().unpack(), is_first_pull);
@ -224,8 +226,8 @@ void ConfidenceCalculator::pullProcessedData(const std::vector<std::string>& fil
m_confidence_level = getConfFile.getConfidenceLevels().unpackMove();
}
}
// is_first_pull = false -> at least one file was downloaded and merged
if (is_first_pull) {
// is_ok = false -> no file was downloaded and merged
if (!is_ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state";
}
}

2
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc
View File

@ -151,9 +151,9 @@ void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std
bool res = sendObjectWithRetry(getTrustFile,
I_Messaging::Method::GET,
getUri() + "/" + file);
pull_ok |= res;
if (res && getTrustFile.getTrustedLogs().ok()) {
mergeFromRemote(getTrustFile.getTrustedLogs().unpack());
pull_ok = true;
}
}
if (!pull_ok && !files.empty()) {

18
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -101,6 +101,24 @@ inline unsigned char from_hex(unsigned char ch, bool &valid) {
return ch;
}
inline bool str_starts_with(const std::string& value, const std::string& prefix)
{
if (prefix.size() > value.size()) {
return false;
}
return value.compare(0, prefix.size(), prefix) == 0;
}
inline bool str_ends_with(const std::string& value, const std::string& ending)
{
if (ending.size() > value.size()) {
return false;
}
return value.compare(value.size() - ending.size(), ending.size(), ending) == 0;
}
template<class _IT>
_IT unquote_plus(_IT first, _IT last, bool decodeUrl=true, bool decodePlus=true) {
_IT result = first;

4
components/security_apps/waap/waap_component_impl.cc
View File

@ -89,6 +89,8 @@ WaapComponent::Impl::init(const std::string &waapDataFileName)
reputationAggregator.init();
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
bool success = waf2_proc_start(waapDataFileName);
if (!success) {
dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!";
@ -101,8 +103,6 @@ WaapComponent::Impl::init(const std::string &waapDataFileName)
I_StaticResourcesHandler *static_resources = Singleton::Consume<I_StaticResourcesHandler>::by<WaapComponent>();
static_resources->registerStaticResource("cp-ab.js", "/etc/cp/conf/waap/cp-ab.js");
static_resources->registerStaticResource("cp-csrf.js", "/etc/cp/conf/waap/cp-csrf.js");
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
}
// Called when component is shut down

64
components/signal_handler/signal_handler.cc
View File

@ -28,17 +28,11 @@
#include <chrono>
#include <boost/regex.hpp>
#if defined(use_unwind)
#if defined(alpine) || defined(PLATFORM_x86)
#ifdef UNWIND_LIBRARY
#define UNW_LOCAL_ONLY
#include <cxxabi.h>
#include <libunwind.h>
#endif // defined(alpine) || defined(PLATFORM_x86)
#endif // defined(use_unwind)
#if !defined(alpine) && !defined(arm32_musl)
#include <execinfo.h>
#endif // not alpine && not arm32_musl
#endif // UNWIND_LIBRARY
#include "debug.h"
#include "common.h"
@ -82,11 +76,9 @@ public:
Maybe<vector<string>>
getBacktrace() override
{
#ifdef UNWIND_LIBRARY
vector<string> symbols;
#if defined(_UCLIBC_) || defined(arm32_musl) || !defined(use_unwind)
return genError("Could not print any backtrace entries using uclibc (backtrace_symbols not supported)");
#else // not (_UCLIBC_ || arm32_musl)
#if defined(alpine) || defined(PLATFORM_x86)
unw_cursor_t cursor;
unw_context_t context;
@ -116,24 +108,14 @@ public:
}
symbols.push_back(buf);
}
#else // not (alpine || PLATFORM_x86)
auto stack_trace_list = vector<void *>(stack_trace_max_len);
uint trace_len = backtrace(stack_trace_list.data(), stack_trace_list.size());
if (trace_len == 0 ) return genError("Could not find any backtrace entries in the current process");
char **trace_prints = backtrace_symbols(stack_trace_list.data(), trace_len);
if (trace_prints == nullptr) return genError("Could not convert backtrace entries to symbol strings");
symbols.reserve(trace_len);
for (uint i = 0; i < trace_len; ++i) {
symbols.emplace_back(trace_prints[i]);
}
free(trace_prints);
#endif // alpine || PLATFORM_x86
#endif // _UCLIBC_
return symbols;
#else // UNWIND_LIBRARY
return genError("Could not print any backtrace entries using uclibc (backtrace_symbols not supported)");
#endif // UNWIND_LIBRARY
}
private:
@ -388,18 +370,13 @@ private:
static void
printStackTrace()
{
#ifdef UNWIND_LIBRARY
if (out_trace_file_fd == -1) return;
const char *stack_trace_title = "Stack trace:\n";
writeData(stack_trace_title, strlen(stack_trace_title));
#if defined(_UCLIBC_) || defined(arm32_musl) || !defined(use_unwind)
const char *uclibc_error =
"Could not print any backtrace entries using uclibc (backtrace_symbols not supported)\n";
writeData(uclibc_error, strlen(uclibc_error));
return;
#else // not (_UCLIBC_ || arm32_musl)
#ifdef alpine
unw_cursor_t cursor;
unw_context_t uc;
unw_getcontext(&uc);
@ -426,19 +403,14 @@ private:
if (unw_step(&cursor) <= 0) return;
}
#else // not alpine
void *stack_trace_list[stack_trace_max_len];
uint actual_trace_len = backtrace(stack_trace_list, stack_trace_max_len);
if (actual_trace_len == 0 ) {
const char *no_bt_found_error = "Could not find any backtrace entries in the current process\n";
writeData(no_bt_found_error, strlen(no_bt_found_error));
return;
}
#else // UNWIND_LIBRARY
backtrace_symbols_fd(stack_trace_list, actual_trace_len, out_trace_file_fd);
#endif // alpine
#endif // _UCLIBC_ || arm32_musl
const char *uclibc_error =
"Could not print any backtrace entries using uclibc (backtrace_symbols not supported)\n";
writeData(uclibc_error, strlen(uclibc_error));
#endif // UNWIND_LIBRARY
}
// LCOV_EXCL_STOP

71
components/utils/http_transaction_data/http_transaction_data.cc
View File

@ -101,7 +101,7 @@ deserializeStrParam(const Buffer &data, uint &cur_pos)
<< to_string(*str_size);
cur_pos += *str_size;
return move(res);
}
@ -185,6 +185,39 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data)
dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized client port: " << client_port.unpack();
}
if (cur_pos == transaction_raw_data.size()) {
dbgDebug(D_NGINX_ATTACHMENT)
<< "No extra data to read from buffer. This agent is working with an old "
<< "attachment version that does not contain the parsed host and parsed uri elements.";
HttpTransactionData transaction(
http_protocol.unpackMove(),
http_method.unpackMove(),
host_name.unpackMove(),
listening_addr.unpackMove(),
listening_port.unpackMove(),
uri.unpackMove(),
client_addr.unpackMove(),
client_port.unpackMove()
);
return transaction;
}
Maybe<string> ngx_parsed_host = deserializeStrParam(transaction_raw_data, cur_pos);
if (!ngx_parsed_host.ok()) {
return genError("Could not deserialize nginx host: " + ngx_parsed_host.getErr());
} else {
dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized nginx_host: " << ngx_parsed_host.unpack();
}
Maybe<string> ngx_parsed_uri = deserializeStrParam(transaction_raw_data, cur_pos);
if (!ngx_parsed_uri.ok()) {
return genError("Could not deserialize parsed URI: " + ngx_parsed_uri.getErr());
} else {
dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized parsed URI: " << ngx_parsed_uri.unpack();
}
// Fail if after parsing exact number of items, we didn't exactly consume whole buffer
if (cur_pos != transaction_raw_data.size()) {
dbgWarning(D_NGINX_ATTACHMENT) << "Nothing to deserialize, but raw data still remain";
@ -195,14 +228,16 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data)
http_protocol.unpackMove(),
http_method.unpackMove(),
host_name.unpackMove(),
ngx_parsed_host.unpackMove(),
listening_addr.unpackMove(),
listening_port.unpackMove(),
uri.unpackMove(),
ngx_parsed_uri.unpackMove(),
client_addr.unpackMove(),
client_port.unpackMove()
);
return move(transaction);
return transaction;
}
HttpTransactionData::HttpTransactionData (
@ -214,14 +249,44 @@ HttpTransactionData::HttpTransactionData (
string _uri,
IPAddr _client_ip,
uint16_t _client_port
)
:
HttpTransactionData::HttpTransactionData(
_http_proto,
_method,
_host_name,
_host_name,
_listening_ip,
_listening_port,
_uri,
_uri,
_client_ip,
_client_port
)
{
}
HttpTransactionData::HttpTransactionData (
string _http_proto,
string _method,
string _host_name,
string _parsed_host,
IPAddr _listening_ip,
uint16_t _listening_port,
string _uri,
string _parsed_uri,
IPAddr _client_ip,
uint16_t _client_port
)
:
http_proto(move(_http_proto)),
method(move(_method)),
host_name(move(_host_name)),
parsed_host(move(_parsed_host)),
listening_ip(move(_listening_ip)),
listening_port(move(_listening_port)),
uri(move(_uri)),
parsed_uri(move(_parsed_uri)),
client_ip(move(_client_ip)),
client_port(move(_client_port)),
is_request(true),
@ -235,9 +300,11 @@ HttpTransactionData::HttpTransactionData()
"",
"GET",
"",
"",
IPAddr(),
-1,
"",
"",
IPAddr(),
-1
)

44
components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc
View File

@ -102,6 +102,8 @@ TEST_F(HttpTransactionTest, TestTransactionDataFromBuf)
EXPECT_EQ(data.getHttpProtocol(), "HTTP/1.1");
EXPECT_EQ(data.getURI(), "/user-app/");
EXPECT_EQ(data.getHttpMethod(), "GET");
EXPECT_EQ(data.getParsedURI(), "/user-app/");
EXPECT_EQ(data.getParsedHost(), "localhost");
}
TEST_F(HttpTransactionTest, TestTransactionDataBadVer)
@ -125,3 +127,45 @@ TEST_F(HttpTransactionTest, TestTransactionDataBadAddress)
"Could not parse IP Address: String 'this.is.not.IP' is not a valid IPv4/IPv6 address"
);
}
TEST_F(HttpTransactionTest, TestTransactionDataFromBufWIthParsedHostAndParsedUri)
{
Buffer meta_data =
Buffer(encodeInt16(strlen("HTTP/1.1"))) +
Buffer("HTTP/1.1") +
encodeInt16(3) +
Buffer("GET") +
encodeInt16(9) +
Buffer("localhost") +
encodeInt16(7) +
Buffer("0.0.0.0") +
encodeInt16(443) +
encodeInt16(11) +
Buffer("//user-app/") +
encodeInt16(9) +
Buffer("127.0.0.1") +
encodeInt16(47423) +
encodeInt16(10) +
Buffer("localhost2") +
encodeInt16(10) +
Buffer("/user-app/");
HttpTransactionData data = HttpTransactionData::createTransactionData(meta_data).unpack();
stringstream data_stream;
data.print(data_stream);
string data_string(
"HTTP/1.1 GET\nFrom: 127.0.0.1:47423\nTo: localhost//user-app/ (listening on 0.0.0.0:443)\n"
);
EXPECT_EQ(data_stream.str(), data_string);
EXPECT_EQ(data.getSourceIP(), IPAddr::createIPAddr("127.0.0.1").unpack());
EXPECT_EQ(data.getSourcePort(), 47423);
EXPECT_EQ(data.getListeningIP(), IPAddr::createIPAddr("0.0.0.0").unpack());
EXPECT_EQ(data.getListeningPort(), 443);
EXPECT_EQ(data.getDestinationHost(), "localhost");
EXPECT_EQ(data.getHttpProtocol(), "HTTP/1.1");
EXPECT_EQ(data.getURI(), "//user-app/");
EXPECT_EQ(data.getHttpMethod(), "GET");
EXPECT_EQ(data.getParsedURI(), "/user-app/");
EXPECT_EQ(data.getParsedHost(), "localhost2");
}

58
core/config/config.cc
View File

@ -35,6 +35,8 @@ using namespace Config;
USE_DEBUG_FLAG(D_CONFIG);
static const string not_found = "";
AgentProfileSettings AgentProfileSettings::default_profile_settings = AgentProfileSettings();
class registerExpectedConfigUpdates : public ClientRest
@ -107,7 +109,7 @@ public:
PerContextValue getAllConfiguration(const std::vector<std::string> &paths) const;
const TypeWrapper & getResource(const vector<string> &paths) const override;
const TypeWrapper & getSetting(const vector<string> &paths) const override;
const string & getProfileAgentSetting(const string &setting_name) const override;
string getProfileAgentSetting(const string &setting_name) const override;
vector<string> getProfileAgentSettings(const string &regex) const override;
const string & getConfigurationFlag(const string &flag_name) const override;
@ -139,9 +141,9 @@ public:
void registerConfigPrepareCb(ConfigCb) override;
void registerConfigLoadCb(ConfigCb) override;
void registerConfigAbortCb(ConfigCb) override;
void clearOldTenants() override;
private:
void clearOldTenants();
bool areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const;
void periodicRegistrationRefresh();
@ -152,6 +154,7 @@ private:
void reloadConfigurationContinuesWrapper(const string &version, uint id);
vector<string> fillMultiTenantConfigFiles(const map<string, set<string>> &tenants);
vector<string> fillMultiTenantExpectedConfigFiles(const map<string, set<string>> &tenants);
map<string, string> getProfileAgentSetting() const;
string
getActiveTenant() const
@ -262,7 +265,6 @@ private:
unordered_map<TenantProfilePair, map<vector<string>, PerContextValue>> configuration_nodes;
unordered_map<TenantProfilePair, map<vector<string>, TypeWrapper>> settings_nodes;
map<string, string> profile_settings;
unordered_map<string, string> config_flags;
map<vector<string>, TypeWrapper> new_resource_nodes;
@ -337,13 +339,6 @@ ConfigComponent::Impl::init()
false
);
}
mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
tenant_manager->getTimeoutVal(),
[this] () { clearOldTenants(); },
"Config comp old tenant cleanup"
);
}
static
@ -428,13 +423,13 @@ ConfigComponent::Impl::getSetting(const vector<string> &paths) const
return empty;
}
const string &
string
ConfigComponent::Impl::getProfileAgentSetting(const string &setting_name) const
{
auto profile_settings = getProfileAgentSetting();
auto setting_raw_val = profile_settings.find(setting_name);
if (setting_raw_val != profile_settings.end()) return setting_raw_val->second;
const static string not_found = "";
return not_found;
}
@ -447,7 +442,6 @@ ConfigComponent::Impl::getConfigurationFlag(const string &flag_name) const
flag = config_flags.find(flag_name);
if (flag != config_flags.end()) return flag->second;
const static string not_found = "";
return not_found;
}
@ -535,6 +529,7 @@ ConfigComponent::Impl::getProfileAgentSettings(const string &regex) const
{
vector<string> setting_raw_values;
boost::regex reg(regex);
auto profile_settings = getProfileAgentSetting();
for (auto &setting : profile_settings) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, setting.first, reg)) {
setting_raw_values.push_back(setting.second);
@ -781,23 +776,6 @@ ConfigComponent::Impl::commitSuccess()
configuration_nodes = move(new_configuration_nodes);
settings_nodes = move(new_settings_nodes);
AgentProfileSettings profile_agent_settings = getSettingWithDefault<AgentProfileSettings>(
AgentProfileSettings::default_profile_settings,
"agentSettings"
);
AgentProfileSettings general_agent_settings = getSettingWithDefault<AgentProfileSettings>(
AgentProfileSettings::default_profile_settings,
"generalAgentSettings"
);
auto tmp_general_settings = general_agent_settings.getSettings();
for (const pair<string, string> &profile_setting : profile_agent_settings.getSettings()) {
tmp_general_settings.insert(profile_setting);
}
profile_settings = tmp_general_settings;
reloadFileSystemPaths();
for (auto &cb : configuration_commit_cbs) {
@ -868,14 +846,16 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
map<string, shared_ptr<ifstream>> files;
for (const auto &path : config_file_paths) {
dbgTrace(D_CONFIG) << "Inserting " << path << " to the list of files to be handled";
auto fullpath = config_directory_path + path;
files.emplace(fullpath, make_shared<ifstream>(fullpath));
}
map<string, set<string>> active_tenants =
tenant_manager ? tenant_manager->fetchActiveTenantsAndProfiles() : map<string, set<string>>();
tenant_manager ? tenant_manager->fetchAndUpdateActiveTenantsAndProfiles(true) : map<string, set<string>>();
dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size();
clearOldTenants();
const vector<string> &config_files = fillMultiTenantConfigFiles(active_tenants);
const vector<string> &expected_config_files = fillMultiTenantExpectedConfigFiles(active_tenants);
@ -903,6 +883,22 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
return res;
}
map<string, string>
ConfigComponent::Impl::getProfileAgentSetting() const
{
auto general_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "generalAgentSettings");
auto settings = general_sets.getSettings();
auto profile_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "agentSettings");
auto profile_settings = profile_sets.getSettings();
for (const auto &profile_setting : profile_settings) {
settings.insert(profile_setting);
}
return settings;
}
void
ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version, uint id)
{

6
core/config/config_globals.cc
View File

@ -77,6 +77,12 @@ getFilesystemPathConfig()
return Singleton::Consume<I_Config>::from<MockConfigProvider>()->getFilesystemPathConfig();
}
void
clearOldTenants()
{
Singleton::Consume<I_Config>::from<MockConfigProvider>()->clearOldTenants();
}
const string &
getLogFilesPathConfig()
{

12
core/include/attachments/nginx_attachment_common.h
View File

@ -18,6 +18,7 @@
#include <stddef.h>
#include <stdint.h>
#include <sys/types.h>
#include <assert.h>
#define MAX_NGINX_UID_LEN 32
#define NUM_OF_NGINX_IPC_ELEMENTS 200
@ -183,6 +184,10 @@ typedef enum ngx_http_meta_data
CLIENT_ADDR_SIZE,
CLIENT_ADDR_DATA,
CLIENT_PORT,
PARSED_HOST_SIZE,
PARSED_HOST_DATA,
PARSED_URI_SIZE,
PARSED_URI_DATA,
META_DATA_COUNT
} ngx_http_meta_data_e;
@ -242,6 +247,7 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data {
} custom_response_data;
struct __attribute__((__packed__)) ngx_http_cp_redirect_data {
uint8_t unused_dummy;
uint8_t add_event_id;
uint16_t redirect_location_size;
char redirect_location[0];
@ -249,6 +255,12 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data {
} response_data;
} ngx_http_cp_web_response_data_t;
static_assert(
sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.custom_response_data) ==
sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.redirect_data),
"custom_response_data must be equal to redirect_data in size"
);
typedef union __attribute__((__packed__)) ngx_http_cp_modify_data {
ngx_http_cp_inject_data_t inject_data[0];
ngx_http_cp_web_response_data_t web_response_data[0];

4
core/include/general/intelligence_comp_v2.h
View File

@ -20,6 +20,8 @@
#include "i_messaging.h"
#include "i_mainloop.h"
#include "i_time_get.h"
#include "i_agent_details.h"
#include "i_rest_api.h"
#include "component.h"
class IntelligenceComponentV2
@ -28,6 +30,8 @@ class IntelligenceComponentV2
Singleton::Provide<I_Intelligence_IS_V2>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_TimeGet>
{
public:

4
core/include/general/tenant_manager.h
View File

@ -21,6 +21,7 @@
#include "i_time_get.h"
#include "i_instance_awareness.h"
#include "component.h"
#include "i_shell_cmd.h"
enum class TenantManagerType { CLIENT, SERVER };
@ -33,7 +34,8 @@ class TenantManager
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_RestApi>
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_ShellCmd>
{
public:
TenantManager();

24
core/include/services_sdk/interfaces/i_intelligence_is_v2.h
View File

@ -25,12 +25,25 @@
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "config.h"
namespace Intelligence {
class Invalidation;
} // namespace Intelligence
class I_Intelligence_IS_V2
{
public:
virtual bool sendInvalidation(const Intelligence::Invalidation &invalidation) const = 0;
virtual Maybe<uint> registerInvalidation(
const Intelligence::Invalidation &invalidation,
const std::function<void(const Intelligence::Invalidation &)> &callback
) = 0;
virtual void unregisterInvalidation(uint id) = 0;
template<typename Data>
Maybe<std::vector<AssetReply<Data>>>
queryIntelligence(QueryRequest &query_request)
queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true)
{
uint assets_limit = query_request.getAssetsLimit();
static const uint upper_assets_limit = 50;
@ -50,7 +63,7 @@ public:
return genError("Paging is activated and already finished. No need for more queries.");
}
IntelligenceQuery<Data> intelligence_query(query_request);
IntelligenceQuery<Data> intelligence_query(query_request, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/query";
bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit);
@ -67,12 +80,15 @@ public:
}
}
if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return genError("Query intelligence response with InProgress status");
}
return intelligence_query.getData();
}
template<typename Data>
Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
queryIntelligence(std::vector<QueryRequest> &query_requests)
queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true)
{
static const uint upper_assets_limit = 50;
static const uint upper_confidence_limit = 1000;
@ -95,7 +111,7 @@ public:
);
}
}
IntelligenceQuery<Data> intelligence_query(query_requests);
IntelligenceQuery<Data> intelligence_query(query_requests, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/queries";
dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items";

38
core/include/services_sdk/interfaces/i_messaging_downloader.h
View File

@ -1,38 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __I_MESSAGING_DOWNLOADER_H__
#define __I_MESSAGING_DOWNLOADER_H__
#include <string>
#include <functional>
#include "maybe_res.h"
class I_MessagingDownloader
{
public:
using OnCompleteCB = std::function<void(const Maybe<std::string> &)>;
virtual bool downloadFile(
const std::string &file_name,
const std::string &url,
OnCompleteCB cb = nullptr,
const unsigned int port = 0
) = 0;
protected:
virtual ~I_MessagingDownloader() {}
};
#endif // __I_MESSAGING_DOWNLOADER_H__

2
core/include/services_sdk/interfaces/i_rest_api.h
View File

@ -48,6 +48,8 @@ public:
return addRestCall(oper, uri, std::make_unique<SpecificRestInit<T>>());
}
virtual uint16_t getListeningPort() const = 0;
protected:
~I_RestApi() {}
virtual bool addRestCall(RestAction oper, const std::string &uri, std::unique_ptr<RestInit> &&init) = 0;

8
core/include/services_sdk/interfaces/i_tenant_manager.h
View File

@ -21,12 +21,11 @@
#include <chrono>
#include <functional>
#include "tenant_profile_pair.h"
class I_TenantManager
{
public:
using newTenantCB = std::function<void(const std::vector<std::string> &)>;
virtual void uponNewTenants(const newTenantCB &cb) = 0;
virtual bool areTenantAndProfileActive(const std::string &tenant_id, const std::string &profile_id) const = 0;
virtual std::set<std::string> fetchAllActiveTenants() const = 0;
@ -36,14 +35,13 @@ public:
const std::string &profile_id
) const = 0;
virtual std::map<std::string, std::set<std::string>> fetchActiveTenantsAndProfiles() const = 0;
virtual std::map<std::string, std::set<std::string>> fetchAndUpdateActiveTenantsAndProfiles(bool update) = 0;
virtual std::set<std::string> fetchProfileIds(const std::string &tenant_id) const = 0;
virtual void deactivateTenant(const std::string &tenant_id, const std::string &profile_id) = 0;
virtual void addActiveTenantAndProfile(const std::string &tenant_id, const std::string &profile_id) = 0;
virtual std::chrono::microseconds getTimeoutVal() const = 0;
virtual std::set<std::string> getProfileIdsForRegionAccount(
const std::string &tenant_id,
const std::string &region,

11
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h
View File

@ -27,20 +27,22 @@ template <typename UserSerializableReplyAttr>
class IntelligenceQuery
{
public:
IntelligenceQuery(QueryRequest &filter)
IntelligenceQuery(QueryRequest &filter, bool is_pretty)
:
request(filter),
response(),
responses(),
is_bulk(false)
is_bulk(false),
is_pretty(is_pretty)
{}
IntelligenceQuery(std::vector<QueryRequest> &filters)
IntelligenceQuery(std::vector<QueryRequest> &filters, bool is_pretty)
:
requests(filters),
response(),
responses(),
is_bulk(true)
is_bulk(true),
is_pretty(is_pretty)
{}
Maybe<std::string> genJson() const;
@ -67,6 +69,7 @@ private:
IntelligenceQueryResponse<UserSerializableReplyAttr> response;
std::vector<IntelligenceQueryResponse<UserSerializableReplyAttr>> responses;
bool is_bulk;
bool is_pretty;
};
#include "intelligence_query_v2_impl.h"

11
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h
View File

@ -18,6 +18,9 @@
#error intelligence_query_impl_v2.h should not be included directly!
#endif // __INTELLIGENCE_QUERY_V2_H__
#include <sstream>
#include "json_stream.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
@ -32,9 +35,10 @@ Maybe<std::string>
IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
{
{
std::stringstream out;
std::stringstream str_stream;
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(out);
cereal::JSONOutputArchive out_ar(json_stream);
if (is_bulk) {
std::vector<BulkQueryRequest> bulk_requests;
int index = 0;
@ -46,7 +50,8 @@ IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
request.saveToJson(out_ar);
}
}
return out.str();
return str_stream.str();
}
}

4
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h
View File

@ -43,7 +43,9 @@ enum class Condition
STARTS_WITH,
CONTAINS,
IN,
NOT_IN
NOT_IN,
GREATER_THAN,
LESS_THAN
};
enum class CursorState {

35
core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h Normal file
View File

@ -0,0 +1,35 @@
// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __JSON_STREAM_H__
#define __JSON_STREAM_H__
#include <ostream>
class JsonStream : public std::streambuf, public std::ostream
{
public:
JsonStream(std::ostream *os, bool is_pretty);
private:
int overflow(int c) override;
void emplace(char c);
void add(char c);
std::ostream *os;
bool is_prev_single_backslash = false;
bool is_pretty;
bool in_string = false;
};
#endif // __JSON_STREAM_H__

29
core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h
View File

@ -14,6 +14,7 @@
#ifndef __QUERY_FILTER_V2_H__
#define __QUERY_FILTER_V2_H__
#include <boost/variant.hpp>
#include <string>
#include <chrono>
#include <boost/functional/hash.hpp>
@ -29,6 +30,8 @@ using namespace Intelligence_IS_V2;
class SerializableQueryCondition
{
public:
typedef boost::variant<int64_t, std::string> ValueVariant;
SerializableQueryCondition() {}
SerializableQueryCondition(Condition _condition_type, std::string _key, std::string _value)
@ -36,18 +39,25 @@ public:
condition_type(_condition_type),
key(_key),
value(_value)
{};
{}
SerializableQueryCondition(Condition _condition_type, std::string _key, int64_t _value)
:
condition_type(_condition_type),
key(_key),
value(_value)
{}
void save(cereal::JSONOutputArchive &ar) const;
Condition getConditionType() const { return condition_type; }
const std::string & getKey() const { return key; }
const std::string & getValue() const { return value; }
const ValueVariant & getValue() const { return value; }
private:
Condition condition_type = Condition::EQUALS;
std::string key = "";
std::string value = "";
ValueVariant value = "";
};
class SerializableQueryFilter
@ -55,22 +65,18 @@ class SerializableQueryFilter
public:
SerializableQueryFilter() {}
SerializableQueryFilter(Condition condition_type, const std::string &key, const std::string &value);
SerializableQueryFilter(
Operator operator_type,
Condition condition_type,
const std::string &key,
const std::string &value
);
SerializableQueryFilter(Condition condition_type, const std::string &key, const int64_t &value);
void save(cereal::JSONOutputArchive &ar) const;
void addCondition(Condition condition_type, const std::string &key, const std::string &value);
void addCondition(Condition condition_type, const std::string &key, const int64_t &value);
Operator getOperator() const { return operator_type; }
const std::vector<SerializableQueryCondition> & getConditionOperands() const { return condition_operands; }
const std::vector<SerializableQueryFilter> & getQueriesOperands() const { return queries_operands; }
const std::string & getConditionValueByKey(const std::string &key) const;
Maybe<SerializableQueryCondition::ValueVariant> getConditionValueByKey(const std::string &key) const;
bool empty() const { return condition_operands.empty() && queries_operands.empty(); }
@ -80,7 +86,8 @@ public:
private:
void saveCondition(cereal::JSONOutputArchive &ar) const;
void saveOperation(cereal::JSONOutputArchive &ar) const;
SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type);
bool isOperatorComp(const Operator &oper) const;
SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &oper);
Operator operator_type = Operator::NONE;
std::vector<SerializableQueryFilter> queries_operands = {};

15
core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
View File

@ -37,6 +37,14 @@ public:
AttributeKeyType type = AttributeKeyType::MAIN
);
QueryRequest(
Condition condition_type,
const std::string &key,
const int64_t &value,
bool full_response,
AttributeKeyType type = AttributeKeyType::MAIN
);
void saveToJson(cereal::JSONOutputArchive &ar) const;
void save(cereal::JSONOutputArchive &ar) const;
@ -51,6 +59,13 @@ public:
AttributeKeyType attribute_type = AttributeKeyType::MAIN
);
void addCondition(
Condition condition_type,
const std::string &key,
const int64_t &value,
AttributeKeyType attribute_type = AttributeKeyType::MAIN
);
void setRequestedAttr(
const std::string &attr,
AttributeKeyType attribute_type = AttributeKeyType::REGULAR

1
core/include/services_sdk/interfaces/mock/mock_rest_api.h
View File

@ -8,6 +8,7 @@
class MockRestApi : public Singleton::Provide<I_RestApi>::From<MockProvider<I_RestApi>>
{
public:
MOCK_CONST_METHOD0(getListeningPort, uint16_t());
// You can't mock a function with an R-value reference. So mock a slightly different one
MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr<RestInit> &));

6
core/include/services_sdk/interfaces/mock/mock_tenant_manager.h
View File

@ -13,8 +13,6 @@
class MockTenantManager : public Singleton::Provide<I_TenantManager>::From<MockProvider<I_TenantManager>>
{
public:
MOCK_METHOD1(uponNewTenants, void(const I_TenantManager::newTenantCB &cb));
MOCK_CONST_METHOD0(fetchActiveTenantsAndProfiles, std::map<std::string, std::set<std::string>>());
MOCK_CONST_METHOD0(fetchActiveTenants, std::set<std::string>());
MOCK_CONST_METHOD0(fetchAllActiveTenants, std::set<std::string>());
@ -26,13 +24,11 @@ public:
MOCK_CONST_METHOD2(areTenantAndProfileActive, bool(const std::string &, const std::string &));
MOCK_METHOD2(addActiveTenantAndProfile, void(const std::string &, const std::string &));
MOCK_METHOD2(deactivateTenant, void(const std::string &, const std::string &));
MOCK_METHOD1(fetchAndUpdateActiveTenantsAndProfiles, std::map<std::string, std::set<std::string>>(bool));
MOCK_CONST_METHOD3(
getProfileIdsForRegionAccount,
std::set<std::string>(const std::string &, const std::string &, const std::string &)
);
MOCK_CONST_METHOD0(getTimeoutVal, std::chrono::microseconds());
private:
MOCK_METHOD3(
addInstance,

1
core/include/services_sdk/resources/config.h
View File

@ -90,6 +90,7 @@ std::string getConfigurationFlagWithDefault(const std::string &default_val, cons
const std::string & getFilesystemPathConfig();
const std::string & getLogFilesPathConfig();
void clearOldTenants();
std::string getPolicyConfigPath(
const std::string &name,

2
core/include/services_sdk/resources/config/config_impl.h
View File

@ -155,7 +155,7 @@ Maybe<SettingType, Config::Errors>
getProfileAgentSetting(const std::string &setting)
{
auto i_config = Singleton::Consume<Config::I_Config>::from<Config::MockConfigProvider>();
const std::string &value = i_config->getProfileAgentSetting(setting);
std::string value = i_config->getProfileAgentSetting(setting);
if (value.empty()) return TypeWrapper::failMissing<SettingType>();

4
core/include/services_sdk/resources/config/i_config.h
View File

@ -39,7 +39,7 @@ public:
virtual PerContextValue getAllConfiguration(const std::vector<std::string> &paths) const = 0;
virtual const TypeWrapper & getResource(const std::vector<std::string> &paths) const = 0;
virtual const TypeWrapper & getSetting(const std::vector<std::string> &paths) const = 0;
virtual const string & getProfileAgentSetting(const string &setting_name) const = 0;
virtual string getProfileAgentSetting(const string &setting_name) const = 0;
virtual vector<string> getProfileAgentSettings(const string &setting_name_regex) const = 0;
virtual const string & getConfigurationFlag(const string &flag_name) const = 0;
@ -104,6 +104,8 @@ public:
virtual void registerConfigLoadCb(ConfigCb) = 0;
virtual void registerConfigAbortCb(ConfigCb) = 0;
virtual void clearOldTenants() = 0;
protected:
virtual ~I_Config() {}
};

2
core/include/services_sdk/resources/debug_flags.h
View File

@ -148,6 +148,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_REST_AUTHENTICATION, D_IOT_AUXILIARY)
DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE)
DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE)
@ -156,6 +157,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_IOT_QUERY_ASSETS, D_IOT_RISK)
DEFINE_FLAG(D_IOT_INDICATOR_DATA, D_IOT_RISK)
DEFINE_FLAG(D_IOT_INDICATORS, D_IOT_RISK)
DEFINE_FLAG(D_IOT_DOCKER_WATCHDOG, D_IOT_RISK)
DEFINE_FLAG(D_IOT_DISCOVERY, D_IOT_NEXT)
DEFINE_FLAG(D_IOT_PROBE, D_IOT_DISCOVERY)
DEFINE_FLAG(D_IOT_ASSETS_DATA, D_IOT_DISCOVERY)

74
core/include/services_sdk/resources/intelligence_invalidation.h Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_INVALIDATION_H__
#define __INTELLIGENCE_INVALIDATION_H__
#include <functional>
#include <map>
#include <string>
#include <set>
#include "maybe_res.h"
#include "enum_array.h"
class I_Intelligence_IS_V2;
namespace Intelligence
{
enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION };
class Invalidation
{
public:
Invalidation(const std::string &class_value);
Invalidation & setClassifier(ClassifierType type, const std::string &val);
Invalidation & setStringAttr(const std::string &attr, const std::string &val);
Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val);
Invalidation & setSourceId(const std::string &id);
Invalidation & setObjectType(ObjectType type);
std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
Maybe<std::string, void> getStringAttr(const std::string &attr) const;
Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
const Maybe<std::string, void> & getSourceId() const { return source_id; }
const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
bool report(I_Intelligence_IS_V2 *interface) const;
Maybe<uint> startListening(I_Intelligence_IS_V2 *interface, const std::function<void(const Invalidation &)> &cb);
void stopListening(I_Intelligence_IS_V2 *interface);
Maybe<std::string> genJson() const;
std::string genObject() const;
bool isLegalInvalidation() const;
bool matches(const Invalidation &other) const;
private:
bool hasAttr(const std::string &key, const std::string &value) const;
EnumArray<ClassifierType, std::string, 6> classifiers;
std::map<std::string, std::string> string_main_attr;
std::map<std::string, std::set<std::string>> set_string_main_attr;
Maybe<std::string, void> source_id;
Maybe<ObjectType, void> object_type;
Maybe<uint, void> listening_id;
};
} // namespace Intelligence
#endif // __INTELLIGENCE_INVALIDATION_H__

57
core/include/services_sdk/resources/report/base_field.h
View File

@ -25,8 +25,6 @@
#include "debug.h"
#include "flags.h"
#include "config.h"
#include "virtual_container.h"
#include "Log_modifiers.h"
enum class LogFieldOption { XORANDB64, COUNT };
@ -111,14 +109,53 @@ class LogField : Singleton::Consume<I_Environment>
getSyslogAndCef() const override
{
std::string value(Details::getValueAsString(getValue()));
auto modifier1 = makeVirtualContainer<LogModifiers::ReplaceBackslash>(value);
auto modifier2 = makeVirtualContainer<LogModifiers::ReplaceCR>(modifier1);
auto modifier3 = makeVirtualContainer<LogModifiers::ReplaceLF>(modifier2);
auto modifier4 = makeVirtualContainer<LogModifiers::ReplaceDoubleOuotes>(modifier3);
auto modifier5 = makeVirtualContainer<LogModifiers::ReplaceQuote>(modifier4);
auto modifier6 = makeVirtualContainer<LogModifiers::ReplaceClosingBrace>(modifier5);
auto modifier7 = makeVirtualContainer<LogModifiers::ReplaceEqualSign>(modifier6);
return name + "=\"" + std::string(modifier7.begin(), modifier7.end()) + "\"";
std::string encoded_value;
encoded_value.reserve(value.size() + 6);
for (char ch : value) {
switch (ch) {
case '\\': {
encoded_value.push_back('\\');
encoded_value.push_back('\\');
break;
}
case '\n': {
encoded_value.push_back('\\');
encoded_value.push_back('n');
break;
}
case '\r': {
encoded_value.push_back('\\');
encoded_value.push_back('r');
break;
}
case '"': {
encoded_value.push_back('\\');
encoded_value.push_back('"');
break;
}
case '\'': {
encoded_value.push_back('\\');
encoded_value.push_back('\'');
break;
}
case ']': {
encoded_value.push_back('\\');
encoded_value.push_back(']');
break;
}
case '=': {
encoded_value.push_back('\\');
encoded_value.push_back('=');
break;
}
default: {
encoded_value.push_back(ch);
}
}
}
return name + "=\"" + encoded_value + "\"";
}
// LCOV_EXCL_START Reason: seems that assert prevent the LCOV from identifying that method was tested

2
core/include/services_sdk/resources/report/report_enums.h
View File

@ -63,6 +63,8 @@ enum class Tags {
DEPLOYMENT_K8S,
LAYER_7_ACCESS_CONTROL,
HORIZON_TELEMETRY_METRICS,
CROWDSEC,
PLAYGROUND,
COUNT
};

44
core/include/services_sdk/utilities/customized_cereal_multimap.h
View File

@ -20,6 +20,8 @@
#include "cereal/types/vector.hpp"
#include "cereal/archives/json.hpp"
#include "rest/schema_printer.h"
template <typename ... Types>
class SerializableMultiMap : std::map<std::string, Types> ...
{
@ -40,6 +42,27 @@ public:
}
}
void clear() { clear(Types()...); }
template <typename T>
std::map<std::string, T> &
getMap()
{
return static_cast<std::map<std::string, T> &>(*this);
}
void
performOutputingSchema(std::ostream &out, int level)
{
RestHelper::printIndent(out, level) << "\"additionalProperties\": {\n";
RestHelper::printIndent(out, level + 1) << "\"anyOf\": [";
printTypes<Types...>(out, level +2, 0);
out << '\n';
RestHelper::printIndent(out, level + 1) << "]\n";
RestHelper::printIndent(out, level) << "}";
}
private:
template <typename Archive, typename T, typename ... More>
void
load(const std::string &key, Archive &archive, T t, More ... more)
@ -59,20 +82,29 @@ public:
std::map<std::string, T>::operator[](key) = t;
}
void clear() { clear(Types()...); }
template <typename T, typename ... More>
void clear(const T &t, const More & ... more) { clear(t); clear(more...); }
template <typename T>
void clear(const T &) { std::map<std::string, T>::clear(); }
template <typename T, typename ... More>
void
printTypes(std::ostream &out, int level, uint)
{
printTypes<T>(out, level, 0);
out << ",";
printTypes<More...>(out, level, 0);
}
template <typename T>
std::map<std::string, T> &
getMap()
void
printTypes(std::ostream &out, int level, int)
{
return static_cast<std::map<std::string, T> &>(*this);
out << '\n';
RestHelper::printIndent(out, level) << "{\n";
TypeDector<T>::type(out, level + 1);
RestHelper::printIndent(out, level) << "}";
}
};

14
core/intelligence_is_v2/CMakeLists.txt
View File

@ -1,7 +1,7 @@
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc
)
add_subdirectory(intelligence_is_v2_ut)
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
)
add_subdirectory(intelligence_is_v2_ut)

386
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@ -19,13 +19,200 @@
#include "config.h"
#include "table.h"
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_invalidation.h"
using namespace std;
using namespace chrono;
using namespace Intelligence_IS_V2;
using namespace Intelligence;
USE_DEBUG_FLAG(D_INTELLIGENCE);
static const string primary_port_setting = "local intelligence server primary port";
static const string secondary_port_setting = "local intelligence server secondary port";
static const string invalidation_uri = "/api/v2/intelligence/invalidation";
static const string registration_uri = "/api/v2/intelligence/invalidation/register";
class I_InvalidationCallBack
{
public:
virtual void performCallBacks(const Invalidation &invalidation) const = 0;
protected:
virtual ~I_InvalidationCallBack() {}
};
using MainAttrTypes = SerializableMultiMap<string, set<string>>;
static const map<string, Intelligence::ObjectType> object_names = {
{ "asset", Intelligence::ObjectType::ASSET },
{ "zone", Intelligence::ObjectType::ZONE },
{ "policyPackage", Intelligence::ObjectType::POLICY_PACKAGE },
{ "configuration", Intelligence::ObjectType::CONFIGURATION },
{ "session", Intelligence::ObjectType::SESSION }
};
class InvalidationRegistration
{
public:
class RestCall
{
public:
RestCall(const stringstream &input) : rest_body(input.str()) {}
Maybe<string> genJson() const { return rest_body; }
ostream & print(ostream &os) { return os << rest_body; }
private:
string rest_body;
};
void
addInvalidation(const Invalidation &invalidation)
{
if (!first) stream << ',';
stream << ' ' << invalidation.genObject();
first = false;
}
RestCall
genJson() const
{
stringstream res;
res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", ";
auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
res << "\"name\": \"" << details->getAgentId() << "\", ";
auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
res << "\"dataMap\": [";
res << stream.str();
res << " ] }";
return res;
}
private:
bool first = true;
stringstream stream;
};
class InvalidationCallBack : Singleton::Provide<I_InvalidationCallBack>::SelfInterface
{
public:
uint
emplace(const Invalidation &invalidation, function<void(const Invalidation &)> cb)
{
dbgDebug(D_INTELLIGENCE) << "registering " << invalidation.genObject();
do {
++running_id;
} while (callbacks.find(running_id) != callbacks.end());
callbacks.emplace(running_id, make_pair(invalidation, cb));
return running_id;
}
void erase(uint id) { callbacks.erase(id); }
bool empty() const { return callbacks.empty(); }
InvalidationRegistration::RestCall
getRegistration() const
{
InvalidationRegistration registration;
for (auto &registed_invalidation : callbacks) {
registration.addInvalidation(registed_invalidation.second.first);
}
return registration.genJson();
}
void
performCallBacks(const Invalidation &invalidation) const override
{
for (auto &registed_invalidation : callbacks) {
performCallBacksImpl(invalidation, registed_invalidation.second);
}
}
private:
void
performCallBacksImpl(
const Invalidation &actual_invalidation,
const pair<Invalidation, function<void(const Invalidation &)>> &invalidation_and_cb
) const
{
auto &registereed_invalidation = invalidation_and_cb.first;
auto &cb = invalidation_and_cb.second;
if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation);
}
map<uint, pair<Invalidation, function<void(const Invalidation &)>>> callbacks;
uint running_id = 0;
};
class RecieveInvalidation : public ServerRest
{
public:
void
doCall() override
{
Invalidation invalidation(class_name);
if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get());
if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get());
if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get());
if (order.isActive()) invalidation.setClassifier(ClassifierType::ORDER, order.get());
if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get());
if (mainAttributes.isActive()) {
auto strings = getMainAttr<string>();
for (const auto &value : strings) {
invalidation.setStringAttr(value.first, value.second);
}
auto string_sets = getMainAttr<set<string>>();
for (const auto &value : string_sets) {
invalidation.setStringSetAttr(value.first, value.second);
}
}
if (objectType.isActive()) {
auto type = object_names.find(objectType.get());
if (type != object_names.end()) invalidation.setObjectType(type->second);
}
if (sourceId.isActive()) invalidation.setSourceId(sourceId.get());
auto i_cb = Singleton::Consume<I_InvalidationCallBack>::from<InvalidationCallBack>();
i_cb->performCallBacks(invalidation);
}
private:
template <typename ValType>
map<string, ValType>
getMainAttr()
{
map<string, ValType> res;
for (auto &vec_entry : mainAttributes.get()) {
for (auto &attr : vec_entry.getMap<ValType>()) {
res[attr.first] = attr.second;
}
}
return res;
}
C2S_LABEL_PARAM(string, class_name, "class");
C2S_OPTIONAL_PARAM(string, category);
C2S_OPTIONAL_PARAM(string, family);
C2S_OPTIONAL_PARAM(string, group);
C2S_OPTIONAL_PARAM(string, order);
C2S_OPTIONAL_PARAM(string, kind);
C2S_OPTIONAL_PARAM(string, objectType);
C2S_OPTIONAL_PARAM(string, sourceId);
C2S_OPTIONAL_PARAM(vector<MainAttrTypes>, mainAttributes);
};
class IntelligenceComponentV2::Impl
:
Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
@ -78,6 +265,37 @@ public:
message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::System,
chrono::minutes(12),
[this] () { sendReccurringInvalidationRegistration(); },
"Sending intelligence invalidation"
);
auto rest_api = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
rest_api->addRestCall<RecieveInvalidation>(RestAction::SET, "new-invalidation/source/invalidation");
}
bool
sendInvalidation(const Invalidation &invalidation) const override
{
if (offline_mode_only) return false;
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
}
Maybe<uint>
registerInvalidation(const Invalidation &invalidation, const function<void(const Invalidation &)> &cb) override
{
if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation");
if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation");
return invalidations.emplace(invalidation, cb);
}
void
unregisterInvalidation(uint id) override
{
invalidations.erase(id);
}
I_Messaging *
@ -102,11 +320,20 @@ public:
getOfflineInfoString(const SerializableQueryFilter &query) const override
{
string ip_attr_key = "mainAttributes.ip";
string identifier_value = move(query.getConditionValueByKey(ip_attr_key));
if (identifier_value == "") {
auto valueVariant = query.getConditionValueByKey(ip_attr_key);
if (!valueVariant.ok()) {
return genError("could not find IP main attribute in the given query.");
}
return offline_intelligence.getValueByIdentifier(identifier_value);
const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
if (const string* identifier_value = boost::get<string>(&value)) {
if (*identifier_value == "") {
return genError("Could not find IP main attribute in the given query.");
}
return offline_intelligence.getValueByIdentifier(*identifier_value);
}
return genError("Value is not of type string.");
}
bool
@ -116,8 +343,157 @@ public:
}
private:
bool
hasLocalIntelligence() const
{
return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
}
bool
sendLocalInvalidation(const Invalidation &invalidation) const
{
dbgFlow(D_INTELLIGENCE) << "Starting local invalidation";
return sendLocalInvalidationImpl(invalidation) || sendGlobalInvalidation(invalidation);
}
bool
sendLocalInvalidationImpl(const Invalidation &invalidation) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
}
return
sendLocalInvalidationImpl(invalidation, *server, primary_port_setting) ||
sendLocalInvalidationImpl(invalidation, *server, secondary_port_setting);
}
bool
sendLocalInvalidationImpl(const Invalidation &invalidation, const string &server, const string &port_setting) const
{
dbgFlow(D_INTELLIGENCE) << "Sending to local intelligence";
auto port = getSetting<uint>("intelligence", port_setting);
if (!port.ok()) {
dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
return false;
}
dbgTrace(D_INTELLIGENCE)
<< "Invalidation value: "
<< (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
return message->sendNoReplyObject(
invalidation,
I_Messaging::Method::POST,
server,
*port,
Flags<MessageConnConfig>(),
invalidation_uri,
getHTTPHeaders(),
nullptr,
MessageTypeTag::INTELLIGENCE
);
}
bool
sendGlobalInvalidation(const Invalidation &invalidation) const
{
dbgFlow(D_INTELLIGENCE) << "Starting global invalidation";
dbgTrace(D_INTELLIGENCE)
<< "Invalidation value: "
<< (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
return message->sendNoReplyObject(
invalidation,
I_Messaging::Method::POST,
invalidation_uri,
getHTTPHeaders(),
nullptr,
true,
MessageTypeTag::INTELLIGENCE
);
}
string
getHTTPHeaders() const
{
auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
auto tenant = details->getTenantId();
if (tenant == "") tenant = "Global";
auto agent = details->getAgentId();
return "X-Tenant-Id: " + tenant + "\r\nX-Source-Id: " + agent;
}
bool
sendRegistration(const Invalidation &invalidation) const
{
if (offline_mode_only) return false;
InvalidationRegistration registration;
registration.addInvalidation(invalidation);
return sendLocalRegistrationImpl(registration.genJson());
}
bool
sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
}
return
sendLocalRegistrationImpl(registration, *server, primary_port_setting) ||
sendLocalRegistrationImpl(registration, *server, secondary_port_setting);
}
bool
sendLocalRegistrationImpl(
const InvalidationRegistration::RestCall &registration,
const string &server,
const string &port_setting
) const
{
dbgFlow(D_INTELLIGENCE) << "Sending to local registration";
auto port = getSetting<uint>("intelligence", port_setting);
if (!port.ok()) {
dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
return false;
}
dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
return message->sendNoReplyObject(
registration,
I_Messaging::Method::POST,
server,
*port,
Flags<MessageConnConfig>(),
registration_uri,
"",
nullptr,
MessageTypeTag::INTELLIGENCE
);
}
void
sendReccurringInvalidationRegistration() const
{
if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}
OfflineIntelligeceHandler offline_intelligence;
bool offline_mode_only = false;
InvalidationCallBack invalidations;
I_Messaging *message = nullptr;
I_TimeGet *timer = nullptr;
I_MainLoop *mainloop = nullptr;
@ -141,8 +517,8 @@ IntelligenceComponentV2::preload()
registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
registerExpectedSetting<string>("intelligence", "local intelligence server ip");
registerExpectedSetting<uint>("intelligence", "local intelligence server secondary port");
registerExpectedSetting<uint>("intelligence", "local intelligence server primary port");
registerExpectedSetting<uint>("intelligence", primary_port_setting);
registerExpectedSetting<uint>("intelligence", secondary_port_setting);
registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy);
}

14
core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt
View File

@ -1,7 +1,7 @@
file(COPY offline_intelligence_files_v2 DESTINATION .)
link_directories(${BOOST_ROOT}/lib)
add_unit_test(
intelligence_is_v2_ut
"query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc"
"intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")
file(COPY offline_intelligence_files_v2 DESTINATION .)
link_directories(${BOOST_ROOT}/lib)
add_unit_test(
intelligence_is_v2_ut
"query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc;invalidation_ut.cc"
"intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")

188
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@ -9,6 +9,7 @@
#include "mock/mock_messaging.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_time_get.h"
#include "mock/mock_rest_api.h"
using namespace std;
using namespace testing;
@ -35,8 +36,18 @@ public:
EXPECT_CALL(
mock_ml,
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _))
.WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _)
).WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
EXPECT_CALL(
mock_ml,
addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(720000000), _, _, _)
).WillRepeatedly(Return(0));
EXPECT_CALL(
mock_rest,
mockRestCall(_, "new-invalidation/source/invalidation", _)
).WillRepeatedly(Return(true));
string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
@ -52,6 +63,7 @@ public:
stringstream debug_output;
StrictMock<MockMainLoop> mock_ml;
StrictMock<MockRestApi> mock_rest;
NiceMock<MockTimeGet> mock_time;
::Environment env;
ConfigComponent conf;
@ -853,3 +865,175 @@ TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
}
TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2)
{
string paging_in_progress_response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset1\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test1\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test1\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" },\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset2\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test2\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test2\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"inProgress\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"efgh\"\n"
"}\n"
);
string paging_done_response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset1\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test1\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test1\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" },\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset2\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test2\",\n"
" \"mainAttributes\": {\n"
" \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
" \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"fake online test2\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"efgh\"\n"
"}\n"
);
EXPECT_CALL(
messaging_mock,
sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE))
.WillOnce(Return(paging_in_progress_response_str))
.WillOnce(Return(paging_done_response_str));
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true, AttributeKeyType::NONE);
request.activatePaging();
request.setAssetsLimit(10);
vector<AssetReply<Profile>> objects_reply;
vector<string> objects_ids;
do {
auto object_result = intell->queryIntelligence<Profile>(request, true);
if (!object_result.ok()) {
if (object_result.getErr() == "Query intelligence response with InProgress status") continue;
break;
}
objects_reply = object_result.unpack();
if (objects_reply.empty()) break;
for (const AssetReply<Profile> &current_object : objects_reply) {
if (current_object.getMainAttributes().empty()) {
continue;
}
const string &id = current_object.getMainAttributes().begin()->second[0];
objects_ids.push_back(id);
}
} while (!request.isPagingFinished());
EXPECT_EQ(objects_ids.size(), 2);
}

151
core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
View File

@ -10,29 +10,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(QueryRequestTestV2, QueryTest)
{
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
SerializableQueryFilter request_query1 = request.getQuery();
vector<SerializableQueryCondition> request_operands1 = request_query1.getConditionOperands();
SerializableQueryCondition request_condition = *request_operands1.begin();
EXPECT_EQ(request_query1.getOperator(), Operator::NONE);
EXPECT_EQ(request_condition.getKey(), "mainAttributes.phase");
EXPECT_EQ(request_condition.getValue(), "testing");
request.addCondition(Condition::EQUALS, "user1", "Omry");
request.addCondition(Condition::EQUALS, "user2", "Max");
SerializableQueryFilter request_query2 = request.getQuery();
vector<SerializableQueryCondition> request_operands2 = request_query2.getConditionOperands();
vector<SerializableQueryCondition>::iterator it = request_operands2.begin();
it++;
EXPECT_EQ(request_query2.getOperator(), Operator::AND);
EXPECT_EQ(it->getKey(), "mainAttributes.user1");
EXPECT_EQ(it->getValue(), "Omry");
it++;
EXPECT_EQ(it->getKey(), "mainAttributes.user2");
EXPECT_EQ(it->getValue(), "Max");
string output_json =
"{\n"
@ -67,6 +46,38 @@ TEST(QueryRequestTestV2, QueryTest)
request.saveToJson(out_ar);
}
EXPECT_EQ(out.str(), output_json);
QueryRequest request2(Condition::GREATER_THAN, "prev_time", 1676887025952, true);
request2.addCondition(Condition::LESS_THAN, "curr_time", 1676887025958);
string output_json2=
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"and\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"greaterThan\",\n"
" \"key\": \"mainAttributes.prev_time\",\n"
" \"value\": 1676887025952\n"
" },\n"
" {\n"
" \"operator\": \"lessThan\",\n"
" \"key\": \"mainAttributes.curr_time\",\n"
" \"value\": 1676887025958\n"
" }\n"
" ]\n"
" }\n"
"}";
stringstream out2;
{
cereal::JSONOutputArchive out_ar2(out2);
request2.saveToJson(out_ar2);
}
EXPECT_EQ(out2.str(), output_json2);
}
TEST(QueryRequestTestV2, AttributesTest)
@ -185,6 +196,104 @@ TEST(QueryRequestTestV2, OrQueryTest)
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, AndQueryTestThree)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
QueryRequest and_request_1_2 = request1 && (request2 && request3);
QueryRequest and_request_2_1 = (request1 && request2) && request3;
stringstream out_1_2;
{
cereal::JSONOutputArchive out_1_2_ar(out_1_2);
and_request_1_2.saveToJson(out_1_2_ar);
}
stringstream out_2_1;
{
cereal::JSONOutputArchive out_2_1_ar(out_2_1);
and_request_2_1.saveToJson(out_2_1_ar);
}
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"and\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing1\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing2\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing3\"\n"
" }\n"
" ]\n"
" }\n"
"}";
EXPECT_EQ(out_1_2.str(), output_json);
EXPECT_EQ(out_2_1.str(), output_json);
}
TEST(QueryRequestTestV2, OrQueryTestThree)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
QueryRequest or_request_1_2 = request1 || (request2 || request3);
QueryRequest or_request_2_1 = (request1 || request2) || request3;
stringstream out_1_2;
{
cereal::JSONOutputArchive out_1_2_ar(out_1_2);
or_request_1_2.saveToJson(out_1_2_ar);
}
stringstream out_2_1;
{
cereal::JSONOutputArchive out_2_1_ar(out_2_1);
or_request_2_1.saveToJson(out_2_1_ar);
}
string output_json =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"or\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing1\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing2\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing3\"\n"
" }\n"
" ]\n"
" }\n"
"}";
EXPECT_EQ(out_1_2.str(), output_json);
EXPECT_EQ(out_2_1.str(), output_json);
}
TEST(QueryRequestTestV2, AndWithConditionQueryTest)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

2
core/intelligence_is_v2/intelligence_types_v2.cc
View File

@ -36,6 +36,8 @@ Intelligence_IS_V2::convertConditionTypeToString(const Condition &condition_type
{Condition::CONTAINS, "contains"},
{Condition::IN, "in"},
{Condition::NOT_IN, "notIn"},
{Condition::GREATER_THAN, "greaterThan"},
{Condition::LESS_THAN, "lessThan"},
};
auto condition_str = condition_type_to_string_map.find(condition_type);

240
core/intelligence_is_v2/invalidation.cc Normal file
View File

@ -0,0 +1,240 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_invalidation.h"
#include <sstream>
#include "i_intelligence_is_v2.h"
using namespace Intelligence;
using namespace std;
Invalidation::Invalidation(const string &class_value)
:
source_id(genError<void>()),
object_type(genError<void>()),
listening_id(genError<void>())
{
setClassifier(ClassifierType::CLASS, class_value);
}
Invalidation &
Invalidation::setClassifier(ClassifierType type, const string &val)
{
classifiers[type] = val;
return *this;
}
Invalidation &
Invalidation::setStringAttr(const string &attr, const string &val)
{
string_main_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
{
set_string_main_attr[attr] = val;
return *this;
}
Invalidation &
Invalidation::setSourceId(const string &id)
{
source_id = id;
return *this;
}
Invalidation &
Invalidation::setObjectType(ObjectType type)
{
object_type = type;
return *this;
}
Maybe<string, void>
Invalidation::getStringAttr(const string &attr) const
{
auto val_ref = string_main_attr.find(attr);
if (val_ref == string_main_attr.end()) return genError<void>();
return val_ref->second;
}
Maybe<set<string>, void>
Invalidation::getStringSetAttr(const string &attr) const
{
auto val_ref = set_string_main_attr.find(attr);
if (val_ref == set_string_main_attr.end()) return genError<void>();
return val_ref->second;
}
bool
Invalidation::report(I_Intelligence_IS_V2 *interface) const
{
if (!isLegalInvalidation()) return false;
return interface->sendInvalidation(*this);
}
Maybe<uint>
Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function<void(const Invalidation &)> &cb)
{
auto res = interface->registerInvalidation(*this, cb);
if (res.ok()) listening_id = *res;
return res;
}
void
Invalidation::stopListening(I_Intelligence_IS_V2 *interface)
{
if (listening_id.ok()) interface->unregisterInvalidation(*listening_id);
}
static const map<Intelligence::ObjectType, string> convertObjectType = {
{ Intelligence::ObjectType::ASSET, "asset" },
{ Intelligence::ObjectType::ZONE, "zone" },
{ Intelligence::ObjectType::POLICY_PACKAGE, "policyPackage" },
{ Intelligence::ObjectType::CONFIGURATION, "configuration" },
{ Intelligence::ObjectType::SESSION, "session" }
};
Maybe<string>
Invalidation::genJson() const
{
if (!isLegalInvalidation()) return genError("Incomplete intelligence invalidation");
stringstream invalidation;
invalidation << "{ \"invalidations\": [ " << genObject() <<" ] }";
return invalidation.str();
}
string
Invalidation::genObject() const
{
stringstream invalidation;
invalidation << "{ \"class\": \"" << classifiers[ClassifierType::CLASS] << '"';
if (classifiers[ClassifierType::CATEGORY] != "") {
invalidation <<", \"category\": \"" << classifiers[ClassifierType::CATEGORY] << '"';
}
if (classifiers[ClassifierType::FAMILY] != "") {
invalidation <<", \"family\": \"" << classifiers[ClassifierType::FAMILY] << '"';
}
if (classifiers[ClassifierType::GROUP] != "") {
invalidation <<", \"group\": \"" << classifiers[ClassifierType::GROUP] << '"';
}
if (classifiers[ClassifierType::ORDER] != "") {
invalidation <<", \"order\": \"" << classifiers[ClassifierType::ORDER] << '"';
}
if (classifiers[ClassifierType::KIND] != "") {
invalidation <<", \"kind\": \"" << classifiers[ClassifierType::KIND] << '"';
}
if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
invalidation << ", \"mainAttributes\": [ ";
bool first = true;
for (auto &attr : string_main_attr) {
if (!first) invalidation << ", ";
invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
first = false;
}
for (auto &attr : set_string_main_attr) {
if (!first) invalidation << ", ";
auto val = makeSeparatedStr(attr.second, ", ");
invalidation << "{ \"" << attr.first << "\": [ ";
bool internal_first = true;
for (auto &val : attr.second) {
if (!internal_first) invalidation << ", ";
invalidation << "\"" << val << "\"";
internal_first = false;
}
invalidation << " ] }";
first = false;
}
invalidation << " ]";
}
invalidation << " }";
return invalidation.str();
}
bool
Invalidation::isLegalInvalidation() const
{
if (!set_string_main_attr.empty() || !string_main_attr.empty()) {
if (classifiers[ClassifierType::FAMILY] == "") return false;
}
bool is_prev_empty = false;
for (auto &classifer : classifiers) {
if (is_prev_empty && classifer != "") return false;
is_prev_empty = classifer == "";
}
return true;
}
template <>
class EnumCount<ClassifierType> : public EnumCountSpecialization<ClassifierType, 6> {};
bool
Invalidation::matches(const Invalidation &other) const
{
for (auto key : NGEN::Range<ClassifierType>()) {
if (classifiers[key] != "" && classifiers[key] != other.classifiers[key]) return false;
}
if (object_type.ok()) {
if (!other.object_type.ok() || *object_type != *other.object_type) return false;
}
if (source_id.ok()) {
if (!other.source_id.ok() || *source_id != *other.source_id) return false;
}
for (auto &key_value : string_main_attr) {
if (!other.hasAttr(key_value.first, key_value.second)) return false;
}
for (auto &key_values : set_string_main_attr) {
for (auto &value : key_values.second) {
if (!other.hasAttr(key_values.first, value)) return false;
}
}
return true;
}
bool
Invalidation::hasAttr(const string &key, const string &value) const
{
auto string_elem = string_main_attr.find(key);
if (string_elem != string_main_attr.end()) return string_elem->second == value;
auto set_string_elem = set_string_main_attr.find(key);
if (set_string_elem != set_string_main_attr.end()) {
return set_string_elem->second.find(value) != set_string_elem->second.end();
}
return false;
}

50
core/intelligence_is_v2/json_stream.cc Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/json_stream.h"
JsonStream::JsonStream(std::ostream *os, bool is_pretty) : std::ostream(this), os(os), is_pretty(is_pretty) {}
int
JsonStream::overflow(int c)
{
if (c != std::streambuf::traits_type::eof()) {
add(std::streambuf::traits_type::to_char_type(c));
}
return c;
}
void
JsonStream::emplace(char c)
{
*os << c;
}
void
JsonStream::add(char c)
{
if (is_pretty) {
emplace(c);
return;
}
if (is_prev_single_backslash) {
emplace(c);
is_prev_single_backslash = false;
return;
}
if (c == '"') in_string = !in_string;
if (c == '\\') is_prev_single_backslash = true;
if (in_string || !std::isspace(c)) emplace(c);
}

114
core/intelligence_is_v2/query_filter_v2.cc
View File

@ -23,19 +23,46 @@ using namespace Intelligence_IS_V2;
USE_DEBUG_FLAG(D_INTELLIGENCE);
struct Visitor : public boost::static_visitor<void>
{
Visitor(cereal::JSONOutputArchive& _ar) : ar(_ar) {}
template <typename T>
void operator()(const T& value)
{
ar(cereal::make_nvp("value", value));
}
private:
cereal::JSONOutputArchive &ar;
};
void
SerializableQueryCondition::save(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("operator", string(convertConditionTypeToString(condition_type))),
cereal::make_nvp("key", key),
cereal::make_nvp("value", value)
cereal::make_nvp("key", key)
);
Visitor visitor(ar);
boost::apply_visitor(visitor, value);
}
SerializableQueryFilter::SerializableQueryFilter(Condition condition_type, const string &key, const string &value)
{
condition_operands.push_back(SerializableQueryCondition(condition_type, key, value));
SerializableQueryFilter::SerializableQueryFilter(
Condition condition_type,
const std::string &key,
const std::string &value
) {
condition_operands.emplace_back(condition_type, key, value);
}
SerializableQueryFilter::SerializableQueryFilter(
Condition condition_type,
const std::string &key,
const int64_t &value
) {
condition_operands.emplace_back(condition_type, key, value);
}
void
@ -52,31 +79,28 @@ void
SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const string &value)
{
if (queries_operands.size() > 0) {
SerializableQueryFilter new_query_filter(condition_type, key, value);
queries_operands.push_back(new_query_filter);
queries_operands.emplace_back(condition_type, key, value);
return;
}
if (condition_operands.size() == 1 && operator_type == Operator::NONE) {
operator_type = Operator::AND;
if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
condition_operands.emplace_back(condition_type, key, value);
}
void
SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const int64_t &value)
{
if (queries_operands.size() > 0) {
queries_operands.emplace_back(condition_type, key, value);
return;
}
SerializableQueryCondition cond(condition_type, key, value);
condition_operands.push_back(cond);
if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
condition_operands.emplace_back(condition_type, key, value);
}
void
SerializableQueryFilter::saveCondition(cereal::JSONOutputArchive &ar) const
{
SerializableQueryCondition cond = *condition_operands.begin();
Condition condition_type = cond.getConditionType();
string condition_str = convertConditionTypeToString(condition_type);
string filter_key = cond.getKey();
string filter_value = cond.getValue();
ar(
cereal::make_nvp("operator", condition_str),
cereal::make_nvp("key", filter_key),
cereal::make_nvp("value", filter_value)
);
condition_operands.begin()->save(ar);
}
void
@ -101,7 +125,7 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
}
}
const string &
Maybe<SerializableQueryCondition::ValueVariant>
SerializableQueryFilter::getConditionValueByKey(const string &key) const
{
for (const SerializableQueryCondition &condition : condition_operands) {
@ -110,21 +134,49 @@ SerializableQueryFilter::getConditionValueByKey(const string &key) const
}
}
static string empty_str = "";
return empty_str;
return genError("Key not found.");
}
bool
SerializableQueryFilter::isOperatorComp(const Operator &oper) const
{
return operator_type == Operator::NONE || operator_type == oper;
}
SerializableQueryFilter
SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type)
SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &oper)
{
SerializableQueryFilter query_filter_res;
vector<SerializableQueryFilter> new_queries_operands;
new_queries_operands.push_back(*this);
new_queries_operands.push_back(other_query);
query_filter_res.operator_type = oper;
if (isOperatorComp(oper) && other_query.isOperatorComp(oper)) {
size_t queries_size = queries_operands.size() + other_query.queries_operands.size();
size_t conditions_size = condition_operands.size() + other_query.condition_operands.size();
query_filter_res.queries_operands.reserve(queries_size);
query_filter_res.condition_operands.reserve(conditions_size);
for (const auto &subquery : queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
for (const auto &subquery : other_query.queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : other_query.condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
} else {
query_filter_res.queries_operands.reserve(2);
query_filter_res.queries_operands.push_back(*this);
query_filter_res.queries_operands.push_back(other_query);
}
query_filter_res.queries_operands = new_queries_operands;
query_filter_res.operator_type = operator_type;
return query_filter_res;
}

22
core/intelligence_is_v2/query_request_v2.cc
View File

@ -58,6 +58,18 @@ QueryRequest::QueryRequest(
full_response = full_reponse;
}
QueryRequest::QueryRequest(
Condition condition_type,
const string &key,
const int64_t &value,
bool full_reponse,
AttributeKeyType attribute_type
) {
query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
assets_limit = default_assets_limit;
full_response = full_reponse;
}
Maybe<string>
QueryRequest::convertObjectTypeToString() const
{
@ -139,6 +151,16 @@ QueryRequest::addCondition (
query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
}
void
QueryRequest::addCondition (
Condition condition_type,
const string &key,
const int64_t &value,
AttributeKeyType attribute_type
) {
query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
}
void
QueryRequest::setRequestedAttr(const string &attr, AttributeKeyType attr_type)
{

3
core/logging/cef_stream.cc
View File

@ -58,6 +58,9 @@ CefStream::sendLog(const Report &log)
}
dbgTrace(D_REPORT) << "Connected to socket.";
string cef_report = log.getCef();
if (protocol == I_Socket::SocketType::TCP) {
cef_report = to_string(cef_report.length()) + " " + cef_report;
}
vector<char> data(cef_report.begin(), cef_report.end());
for (size_t tries = 0; tries < 3; tries++) {
if (i_socket->writeData(socket.unpack(), data)) {

47
core/logging/logging_ut/logging_ut.cc
View File

@ -1,4 +1,5 @@
#include "log_generator.h"
#include "log_utils.h"
#include <sstream>
#include <fstream>
@ -466,6 +467,52 @@ TEST_F(LogTest, LogGen)
);
EXPECT_THAT(getMessages(), HasSubstr(str3));
EXPECT_THAT(readLogFile(), HasSubstr(str3));
enum class TestErrors { CPU, MEMORY, DISK };
string str4(
"{\n"
" \"eventTime\": \"0:0:0\",\n"
" \"eventName\": \"Install policy\",\n"
" \"eventSeverity\": \"Info\",\n"
" \"eventPriority\": \"Low\",\n"
" \"eventType\": \"Event Driven\",\n"
" \"eventLevel\": \"Log\",\n"
" \"eventLogLevel\": \"info\",\n"
" \"eventAudience\": \"Internal\",\n"
" \"eventAudienceTeam\": \"\",\n"
" \"eventFrequency\": 0,\n"
" \"eventTags\": [\n"
" \"Policy Installation\"\n"
" ],\n"
" \"eventSource\": {\n"
" \"agentId\": \"Unknown\",\n"
" \"eventTraceId\": \"\",\n"
" \"eventSpanId\": \"\",\n"
" \"issuingEngineVersion\": \"" + Version::getFullVersion() + "\",\n"
" \"serviceName\": \"007\"\n"
" },\n"
" \"eventData\": {\n"
" \"logIndex\": 4,\n"
" \"eventCode\": \"015-0002\"\n"
" }\n"
"}"
);
EXPECT_EQ(
toJson(
LogGen(
"Install policy",
Audience::INTERNAL,
Severity::INFO,
Priority::LOW,
tag1,
Enreachments::BEAUTIFY_OUTPUT
) << ErrorCode<ReportIS::Tags::IOT>::logError(TestErrors::DISK)
),
str4
);
EXPECT_THAT(getMessages(), HasSubstr(str4));
EXPECT_THAT(readLogFile(), HasSubstr(str4));
}
TEST_F(LogTest, LogSpecificStream)

3
core/logging/syslog_stream.cc
View File

@ -51,6 +51,9 @@ void
SyslogStream::sendLog(const Report &log)
{
string syslog_report = log.getSyslog();
if (protocol == I_Socket::SocketType::TCP) {
syslog_report = to_string(syslog_report.length()) + " " + syslog_report;
}
vector<char> data(syslog_report.begin(), syslog_report.end());
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::Offline,

32
core/message/message.cc
View File

@ -160,6 +160,7 @@ private:
bool setSocket();
bool connect(const string &host, const string &overwrite_port);
bool shouldIgnoreSslValidation()const;
bool isBioSocketReady() const;
Maybe<string> calculatePublicKey(const BioUniquePtr<X509> &cert) const;
Maybe<string> getPinnedCertificate();
bool verifyCertPinning(const BioUniquePtr<X509> &cert);
@ -1431,6 +1432,11 @@ MessageConnection::doHandshake(const BioUniquePtr<BIO> &bio)
);
auto end_time = timer->getMonotonicTime() + timeout;
while (timer->getMonotonicTime() < end_time) {
if (!isBioSocketReady()) {
dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
if (mainloop != nullptr) mainloop->yield(true);
continue;
}
if (BIO_do_handshake(bio.get()) > 0 || shouldIgnoreSslValidation()) {
return Maybe<void>();
}
@ -1533,6 +1539,17 @@ MessageConnection::setSocket()
return true;
}
bool
MessageConnection::isBioSocketReady() const
{
#if OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
return BIO_socket_wait(BIO_get_fd(bio.get(), NULL), 0, time(NULL));
#else // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
return true;
#endif // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
}
bool
MessageConnection::connect(const string &host, const string &overwrite_port)
{
@ -1547,6 +1564,11 @@ MessageConnection::connect(const string &host, const string &overwrite_port)
while (timer->getMonotonicTime() < end_time) {
counter++;
if (!isBioSocketReady()) {
dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
if (mainloop != nullptr) mainloop->yield(true);
continue;
}
if (BIO_do_connect(bio.get()) > 0) {
dbgDebug(D_COMMUNICATION)
<< "Successfully established new BIO connection. "
@ -1595,6 +1617,11 @@ MessageConnection::receiveResponse(I_MessageDecoder<T> &decoder)
uint counter = 0;
char buf[1000];
while (timer->getMonotonicTime() < end_time) {
if (!isBioSocketReady()) {
dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
if (mainloop != nullptr) mainloop->yield(true);
continue;
}
int len_or_error_ret_val = BIO_read(bio.get(), buf, sizeof(buf) - 1);
if (len_or_error_ret_val <= 0) {
if (!BIO_should_retry(bio.get())) {
@ -1838,6 +1865,11 @@ MessageConnection::sendData(const string &data) const
int remaining_data_len = data.length();
while (timer->getMonotonicTime() < end_time) {
int offset = data.length() - remaining_data_len;
if (!isBioSocketReady()) {
dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
if (mainloop != nullptr) mainloop->yield(true);
continue;
}
int data_sent_len = BIO_write(bio.get(), data.c_str() + offset, remaining_data_len);
if (data_sent_len > 0) {
if (remaining_data_len - data_sent_len < 0) {

8
core/report/tag_and_enum_management.cc
View File

@ -105,7 +105,9 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
{"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED},
{"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S},
{"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL},
{"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}
{"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS},
{"Crowdsec", ReportIS::Tags::CROWDSEC},
{"Playground", ReportIS::Tags::PLAYGROUND}
};
auto report_is_tag = strings_to_tags.find(tag);
@ -305,7 +307,9 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
"Embedded Deployment",
"Kubernetes Deployment",
"Layer 7 Access Control",
"Horizon Telemetry Metrics"
"Horizon Telemetry Metrics",
"Crowdsec",
"Playground"
};
EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {

2
core/rest/rest_conn.cc
View File

@ -70,7 +70,7 @@ RestConn::parseConn() const
string uri;
os >> uri;
string identifier = uri.substr(uri.find_last_of('/') + 1);
string identifier = uri.substr(uri.find_first_of('/') + 1);
uint len = 0;
while (true) {

5
core/rest/rest_server.cc
View File

@ -47,6 +47,7 @@ public:
bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
uint16_t getListeningPort() const override { return listening_port; }
Maybe<std::string> getSchema(const std::string &uri) const override;
Maybe<std::string> invokeRest(const std::string &uri, istream &in) const override;
@ -60,7 +61,7 @@ private:
I_MainLoop::RoutineID id;
I_MainLoop *mainloop;
map<string, unique_ptr<RestInit>> rest_calls;
uint16_t port_used = 0;
uint16_t listening_port = 0;
vector<uint16_t> port_range;
};
@ -138,7 +139,7 @@ RestServer::Impl::init()
is_primary.ok() && *is_primary
);
uint16_t listening_port = ntohs(addr.sin_port);
listening_port = ntohs(addr.sin_port);
dbgInfo(D_API) << "REST server started: " << listening_port;
Singleton::Consume<I_Environment>::by<RestServer>()->registerValue<int>("Listening Port", listening_port);
};

2
core/rest/rest_ut/rest_config_ut.cc
View File

@ -99,6 +99,8 @@ TEST_F(RestConfigTest, alternative_port_used)
);
mainloop->run();
EXPECT_EQ(Singleton::Consume<I_RestApi>::from(rest_server)->getListeningPort(), *alternative_port);
sa.sin_port = htons(alternative_port.unpack());
EXPECT_EQ(bind(file_descriptor, reinterpret_cast<struct sockaddr *>(&sa), sizeof(struct sockaddr_in)), -1);

35
core/rest/rest_ut/rest_schema_ut.cc
View File

@ -19,6 +19,7 @@
#include <arpa/inet.h>
#include <sstream>
#include "customized_cereal_map.h"
#include "customized_cereal_multimap.h"
using namespace std;
@ -175,6 +176,14 @@ class MustMapInt : public ServerRest
C2S_PARAM(mapStringInt, must_map_int);
};
class MustMultiMap : public ServerRest
{
void doCall() override {}
using mapStringInt = SerializableMultiMap<string, int>;
C2S_PARAM(mapStringInt, must_multimap);
};
TEST(RestSchema, must_map)
{
stringstream string_map_schema;
@ -214,6 +223,32 @@ TEST(RestSchema, must_map)
" ]\n"
"}"
);
stringstream multi_map_schema;
MustMultiMap().performOutputingSchema(multi_map_schema);
EXPECT_EQ(
multi_map_schema.str(),
"{\n"
" \"properties\": {\n"
" \"must_multimap\": {\n"
" \"type\": \"object\",\n"
" \"additionalProperties\": {\n"
" \"anyOf\": [\n"
" {\n"
" \"type\": \"string\"\n"
" },\n"
" {\n"
" \"type\": \"integer\"\n"
" }\n"
" ]\n"
" }\n"
" }\n"
" },\n"
" \"required\": [\n"
" \"must_multimap\"\n"
" ]\n"
"}"
);
}
class MustObject : public ServerRest

159
core/tenant_manager/tenant_manager.cc
View File

@ -24,6 +24,7 @@
#include "hash_combine.h"
using namespace std;
using ProfilesPerTenantMap = map<string, set<string>>;
USE_DEBUG_FLAG(D_TENANT_MANAGER);
@ -75,10 +76,10 @@ public:
void init();
void fini();
void uponNewTenants(const newTenantCB &cb) override;
bool areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const override;
map<string, set<string>> fetchActiveTenantsAndProfiles() const override;
ProfilesPerTenantMap fetchActiveTenantsAndProfiles() const override;
ProfilesPerTenantMap fetchAndUpdateActiveTenantsAndProfiles(bool update) override;
set<string> fetchAllActiveTenants() const override;
set<string> fetchActiveTenants() const override;
set<string> getInstances(const string &tenant_id, const string &profile_id) const override;
@ -88,8 +89,6 @@ public:
void deactivateTenant(const string &tenant_id, const string &profile_id) override;
chrono::microseconds getTimeoutVal() const override;
set<string> getProfileIdsForRegionAccount(
const string &tenant_id,
const string &region,
@ -97,33 +96,24 @@ public:
) const override;
void
addInstance(const string &tenant_id, const string &profile_id, const string &instace_id)
addInstance(const string &tenant_id, const string &profile_id, const string &instace_id) override
{
auto tenant_profile_pair = TenantProfilePair(tenant_id, profile_id);
auto tenant_cache = mapper.find(tenant_profile_pair);
if (tenant_cache == mapper.end()) {
tenant_cache = mapper.insert(make_pair(tenant_profile_pair, TemporaryCache<string, void>())).first;
tenant_cache->second.startExpiration(
getTimeoutVal(),
Singleton::Consume<I_MainLoop>::by<TenantManager>(),
Singleton::Consume<I_TimeGet>::by<TenantManager>()
);
}
tenant_cache->second.createEntry(instace_id);
}
private:
void runUponNewTenants(const vector<string> &new_tenants);
void sendTenantAndProfile(const string &tenant_id, const string &profile_id);
set<string> getAllTenants() const;
set<string> fetchAllProfileIds(const string &tenant_id) const;
set<string> getProfileIds(const string &tenant_id) const;
bool sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port);
TemporaryCache<TenantProfilePair, void> active_tenants;
map<TenantProfilePair, TemporaryCache<string, void>> mapper;
vector<I_TenantManager::newTenantCB> upon_cb;
I_Messaging *i_messaging = nullptr;
TenantManagerType type;
@ -218,111 +208,20 @@ TenantManager::Impl::init()
conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN);
i_messaging = Singleton::Consume<I_Messaging>::by<TenantManager>();
auto cache_timeout = getTimeoutVal();
active_tenants.startExpiration(
cache_timeout,
Singleton::Consume<I_MainLoop>::by<TenantManager>(),
Singleton::Consume<I_TimeGet>::by<TenantManager>()
);
if (type == TenantManagerType::SERVER) {
auto rest = Singleton::Consume<I_RestApi>::by<TenantManager>();
rest->addRestCall<LoadNewTenants>(RestAction::SET, "tenant-id");
rest->addRestCall<FetchActiveTenants>(RestAction::SHOW, "active-tenants");
rest->addRestCall<FetchProfileIds>(RestAction::SHOW, "profile-ids");
}
if (type == TenantManagerType::CLIENT) {
auto interval = chrono::seconds(
getProfileAgentSettingWithDefault<uint32_t>(600, "agentConfig.tenantReportIntervalSeconds")
);
interval = chrono::seconds(
getConfigurationWithDefault(interval.count(), "Tenant Manager", "Report interval")
);
Singleton::Consume<I_MainLoop>::by<TenantManager>()->addRecurringRoutine(
I_MainLoop::RoutineType::System,
interval,
[this] ()
{
auto tenants_ids = fetchActiveTenants();
for (auto tenant_id : tenants_ids) {
auto profile_ids = fetchAllProfileIds(tenant_id);
for (auto profile_id : profile_ids) {
sendTenantAndProfile(tenant_id, profile_id);
}
}
},
"Tenant manager client reporter"
);
}
}
void
TenantManager::Impl::fini()
{
active_tenants.endExpiration();
i_messaging = nullptr;
}
bool
TenantManager::Impl::sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port)
{
if (i_messaging == nullptr) {
i_messaging = Singleton::Consume<I_Messaging>::by<TenantManager>();
}
SendNewTenants new_tenant_and_profile(tenant_id, profile_id);
return i_messaging->sendNoReplyObject(
new_tenant_and_profile,
I_Messaging::Method::POST,
"127.0.0.1",
port,
conn_flags,
"/set-tenant-id"
);
}
void
TenantManager::Impl::runUponNewTenants(const vector<string> &new_tenants)
{
for (auto &cb: upon_cb) {
Singleton::Consume<I_MainLoop>::by<TenantManager>()->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
[this, cb, new_tenants] () { cb(new_tenants); },
"New tenant event handler"
);
}
}
void
TenantManager::Impl::sendTenantAndProfile(const string &tenant_id, const string &profile_id)
{
auto res = sendWithCustomPort(
tenant_id,
profile_id,
getConfigurationWithDefault<uint16_t>(
7777,
"Tenant Manager",
"Orchestrator's primary port"
)
);
if (!res) {
sendWithCustomPort(
tenant_id,
profile_id,
getConfigurationWithDefault<uint16_t>(
7778,
"Tenant Manager",
"Orchestrator's secondary port"
)
);
}
}
set<string>
TenantManager::Impl::getAllTenants() const
{
@ -459,12 +358,6 @@ TenantManager::Impl::getProfileIdsForRegionAccount(
return set<string>();
}
void
TenantManager::Impl::uponNewTenants(const newTenantCB &cb)
{
upon_cb.push_back(cb);
}
bool
TenantManager::Impl::areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const
{
@ -485,24 +378,40 @@ TenantManager::Impl::addActiveTenantAndProfile(const string &tenant_id, const st
<< ", Profile ID: "
<< profile_id;
active_tenants.createEntry(tenant_profile);
if (type == TenantManagerType::CLIENT) {
sendTenantAndProfile(tenant_id, profile_id);
} else {
runUponNewTenants({tenant_id});
}
}
void
TenantManager::Impl::deactivateTenant(const string &tenant_id, const string &profile_id)
{
dbgTrace(D_TENANT_MANAGER)
<< "Deactivate tenant and profile. Tenant ID: "
<< tenant_id
<< ", Profile ID: "
<< profile_id;
active_tenants.deleteEntry(TenantProfilePair(tenant_id, profile_id));
}
map<string, set<string>>
ProfilesPerTenantMap
TenantManager::Impl::fetchAndUpdateActiveTenantsAndProfiles(bool update)
{
if (!update) return fetchActiveTenantsAndProfiles();
active_tenants.clear();
ProfilesPerTenantMap update_active_tenants = fetchActiveTenantsAndProfiles();
for (const auto &tenant_profile_set : update_active_tenants) {
auto tenant_id = tenant_profile_set.first;
for (const auto &profile_id : tenant_profile_set.second) {
active_tenants.createEntry(TenantProfilePair(tenant_id, profile_id));
}
}
return update_active_tenants;
}
ProfilesPerTenantMap
TenantManager::Impl::fetchActiveTenantsAndProfiles() const
{
dbgFlow(D_TENANT_MANAGER) << "Fetching active teants and profiles map";
map<string, set<string>> active_tenants_and_profiles;
ProfilesPerTenantMap active_tenants_and_profiles;
set<string> tenants = fetchAllActiveTenants();
for (const string &tenant : tenants) {
active_tenants_and_profiles[tenant] = fetchProfileIds(tenant);
@ -567,19 +476,6 @@ TenantManager::Impl::fetchProfileIds(const string &tenant_id) const
return (type == TenantManagerType::CLIENT) ? getProfileIds(tenant_id) : fetchAllProfileIds(tenant_id);
}
chrono::microseconds
TenantManager::Impl::getTimeoutVal() const
{
auto cache_timeout = chrono::seconds(
getProfileAgentSettingWithDefault<uint32_t>(900, "Orchestration.TenantTimeoutSeconds")
);
cache_timeout = chrono::seconds(
getConfigurationWithDefault(cache_timeout.count(), "Tenant Manager", "Tenant timeout")
);
return chrono::duration_cast<chrono::microseconds>(cache_timeout);
}
TenantManager::TenantManager()
:
Component("TenantManager"),
@ -604,7 +500,6 @@ TenantManager::fini()
void
TenantManager::preload()
{
registerExpectedConfiguration<uint32_t>("Tenant Manager", "Tenant timeout");
registerExpectedConfiguration<string>("Tenant Manager", "Tenant manager type");
registerExpectedSetting<AccountRegionSet>("accountRegionSet");
registerExpectedSetting<string>("region");

1
nodes/orchestration/CMakeLists.txt
View File

@ -27,7 +27,6 @@ target_link_libraries(
update_communication
orchestration_tools
env_details
messaging_downloader_client
local_policy_mgmt_gen
curl

4
nodes/orchestration/main.cc
View File

@ -17,7 +17,6 @@
#include <algorithm>
#include <unistd.h>
#include "messaging_downloader_client.h"
#include "time_proxy.h"
#include "shell_cmd.h"
#include "debug.h"
@ -58,8 +57,6 @@
using namespace std;
#include "components_list.h"
#include "messaging_downloader_server.h"
#include "external_sdk_server.h"
int
main(int argc, char **argv)
@ -77,7 +74,6 @@ main(int argc, char **argv)
OrchestrationComp,
HealthChecker,
HealthCheckManager,
MessagingDownloaderClient,
LocalPolicyMgmtGenerator
> comps;

9
nodes/orchestration/package/cpnano_debug/cpnano_debug.cc
View File

@ -91,7 +91,6 @@ enum class Service {
CPVIEW_METRIC_PROVIDER,
HTTP_TRANSACTION_HANDLER,
DEDICATED_NETWORK_HANDLER,
MESSAGING_PROXY,
HELLO_WORLD,
IDA,
IOT_ACCESS_CONTROL,
@ -165,7 +164,6 @@ getServiceString(const Service service)
case (Service::ATTACHMENT_REGISTRATOR): return "attachment-registrator";
case (Service::CPVIEW_METRIC_PROVIDER): return "cpview-metric-provider";
case (Service::DEDICATED_NETWORK_HANDLER): return "dedicated-network-handler";
case (Service::MESSAGING_PROXY): return "messaging-proxy";
case (Service::SDWAN): return "sdwan";
case (Service::LOGGER_SDWAN): return "logger-sdwan";
case (Service::IOT_WLP): return "workload-protection";
@ -321,11 +319,6 @@ getServiceConfig (const Service service)
filesystem_path + "/conf/cp-nano-dedicated-network-handler-debug-conf.json",
log_files_path + "/nano_agent/cp-nano-dedicated-network-handler.dbg"
);
case (Service::MESSAGING_PROXY):
return ServiceConfig(
filesystem_path + "/conf/cp-nano-messaging-proxy-debug-conf.json",
log_files_path + "/nano_agent/cp-nano-messaging-proxy.dbg"
);
case (Service::SDWAN):
return ServiceConfig(
filesystem_path + "/conf/cp-nano-sdwan-debug-conf.json",
@ -1282,8 +1275,6 @@ extractServices(const vector<string> &args)
services.push_back(Service::HTTP_TRANSACTION_HANDLER);
} else if (getServiceString(Service::DEDICATED_NETWORK_HANDLER).find(maybe_service) == 0) {
services.push_back(Service::DEDICATED_NETWORK_HANDLER);
} else if (getServiceString(Service::MESSAGING_PROXY).find(maybe_service) == 0) {
services.push_back(Service::MESSAGING_PROXY);
} else if (getServiceString(Service::SDWAN).find(maybe_service) == 0) {
services.push_back(Service::SDWAN);
} else if (getServiceString(Service::LOGGER_SDWAN).find(maybe_service) == 0) {

105
nodes/orchestration/package/open-appsec-ctl.sh
View File

@ -26,6 +26,8 @@ DEFAULT_HEALTH_CHECK_TMP_FILE_PATH="/tmp/cpnano_health_check_output.txt"
var_default_fog_address="i2-agents.cloud.ngen.checkpoint.com/"
var_default_gem_fog_address="inext-agents.cloud.ngen.checkpoint.com"
var_default_us_fog_address="inext-agents-us.cloud.ngen.checkpoint.com"
var_default_au_fog_address="inext-agents-aus1.cloud.ngen.checkpoint.com"
var_default_in_fog_address="inext-agents-ind1.cloud.ngen.checkpoint.com"
#NOTE: open-appsec-ctl only supports nano services with name of the format cp-nano-<service>
cp_nano_service_name_prefix="cp-nano"
@ -92,7 +94,8 @@ fi
get_basename()
{
if command -v basename &>/dev/null; then
is_basename_exist=$(command -v basename)
if [ -n $is_basename_exist ]; then
echo $(basename $1)
else
echo $(echo $1 | rev | cut -d / -f 1 | rev)
@ -267,7 +270,7 @@ usage()
uninstall_option="-u, --uninstall"
load_config_option="-lc, --load-config <$(get_installed_services '|')>"
display_config_option="-dc, --display-config [$(get_installed_services '|')]"
cp_agent_info_option="-ai, --cp-agent-info [-wd|--with_dump|-u|--upload|-fms|--file_max_size|-an|--additional_name]"
cp_agent_info_option="-ai, --cp-agent-info [-wd|--with_dump|-fms|--file_max_size|-an|--additional_name]"
display_policy_option="-dp, --display-policy"
set_gradual_policy_option="-gp, --set-gradual-policy [access-control|http-manager] <ip-ranges>"
delete_gradual_policy_option="-dg, --delete-gradual-policy [access-control|http-manager]"
@ -1217,16 +1220,11 @@ run_ai() # Initials - ra
ra_tenant_id=
ra_agent_id=
ra_token=
ra_upload_to_fog=false
# we use this address as default and replace later if needed
ra_fog_address="inext-agents.cloud.ngen.checkpoint.com"
for arg; do
if [ "$arg" = "--upload" ] || [ "$arg" = "-u" ]; then
ra_upload_to_fog=true
shift
continue
elif [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then
if [ "$arg" = "--verbose" ] || [ "$arg" = "-v" ]; then
AI_VERBOSE=true
elif [ -z "$1" ]; then
break
@ -1235,14 +1233,6 @@ run_ai() # Initials - ra
shift
done
if [ "$ra_upload_to_fog" = "false" ]; then
printf "Should upload to Checkpoints' cloud? [y/n] " && read -r ra_should_upload
case $ra_should_upload in
[Yy] | [Yy][Ee][Ss]) ra_upload_to_fog=true ;;
*) ;;
esac
fi
ra_https_prefix="https://"
ra_agent_details=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/agent_details.json)
if echo "$ra_agent_details" | grep -q "Fog domain"; then
@ -1274,49 +1264,6 @@ run_ai() # Initials - ra
echo "Failed to calculate agent-info data."
exit 1
fi
if [ "$ra_upload_to_fog" = "true" ]; then
ra_token_data="$(curl_func "$(extract_api_port orchestration)"/show-access-token)"
ra_token_hex=$(echo "$ra_token_data" | grep "token" | cut -d '"' -f4 | base64 -d | od -t x1 -An)
ra_token_hex_formatted=$(echo $ra_token_hex | tr -d ' ')
ra_token="$(xor_decrypt "${ra_token_hex_formatted}")"
ra_proxy_val=""
if [ -n "${is_gaia}" ]; then
ra_gaia_proxy_address=$(dbget proxy:ip-address | tr -d '\n')
ra_gaia_proxy_ip=$(dbget proxy:port | tr -d '\n')
if [ -n "$ra_gaia_proxy_address" ] && [ -n "$ra_gaia_proxy_ip" ]; then
ra_proxy_val="--proxy http://${ra_gaia_proxy_address}:${ra_gaia_proxy_ip}"
fi
fi
if [ "$is_smb_release" = "1" ]; then
is_proxy_enabled=$(pt proxySettings | awk '{if ($1 == "useProxy") printf("%s", $3)}')
if [ "$is_proxy_enabled" = "true" ]; then
ra_smb_proxy_address=$(pt proxySettings | awk '{if ($1 == "ipAddress") printf("%s", $3)}')
ra_smb_proxy_port=$(pt proxySettings | awk '{if ($1 == "port") printf("%s", $3)}')
if [ ! -z $ra_smb_proxy_address ] && [ ! -z $ra_smb_proxy_port ]; then
ra_proxy_val="--proxy http://${ra_smb_proxy_address}:${ra_smb_proxy_port}"
fi
fi
fi
echo "---- Uploading agent information to Check Point ----"
sleep 1
upload_ai "$ra_cp_info_path" "$ra_token" "$ra_fog_address" "$ra_tenant_id" "$ra_agent_id" "$ra_current_time" "$ra_file_dir"
if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "true" ]; then
echo "Files are too large - splitting to files of size of $SPLIT_FILE_SMALL_SIZE"
cat "$ra_cp_info_path"/* >"$ra_cp_info_path"/temp_reassembled_files
rm "$ra_cp_info_path"/*.*
split -b "$SPLIT_FILE_SMALL_SIZE" "$ra_cp_info_path"/temp_reassembled_files "$ra_cp_info_path"/cp-nano-info-"$ra_agent_id"-"$ra_current_time".tar.gz
rm "$ra_cp_info_path"/temp_reassembled_files
upload_ai "$ra_cp_info_path" "$ra_token" "$ra_fog_address" "$ra_tenant_id" "$ra_agent_id" "$ra_current_time" "$ra_file_dir"
fi
echo "File upload to cloud: Succeeded"
else
echo "ignore uploading file to the Fog."
fi
}
create_entries_file() # Initials - cef
@ -1346,38 +1293,6 @@ create_entries_file() # Initials - cef
} >>"$cef_entries_file_path"
}
upload_ai() # Initials - uai
{
uai_cp_info_path="$1"
uai_token="$2"
uai_fog_address="$3"
uai_tenant_id="$4"
uai_agent_id="$5"
uai_current_time="$6"
uai_file_dir="$7"
create_entries_file "$uai_cp_info_path"
for file in "$uai_cp_info_path"/*; do
if [ "$AI_VERBOSE" = "true" ]; then
echo "Uploading file $file"
fi
if [ -z "${is_gaia}" -o "$is_smb_release" = "1" ]; then
uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
elif [ "${remove_curl_ld_path}" = "true" ]; then
uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
else
uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
fi
if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "false" ] && [ "$uai_curl_output" = "413" ]; then
AI_UPLOAD_TOO_LARGE_FLAG=true
return
fi
if test "$uai_curl_output" != "200"; then
echo "File upload to cloud: Failed Error code ${uai_curl_output}"
exit 1
fi
done
}
set_mode_usage_message()
{
echo "Usage:"
@ -1464,9 +1379,17 @@ set_mode()
gem_prefix_uppercase="CP-"
us_prefix="cp-us-"
us_prefix_uppercase="CP-US-"
au_prefix="cp-au-"
au_prefix_uppercase="CP-AU-"
in_prefix="cp-in-"
in_prefix_uppercase="CP-IN-"
if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_us_fog_address"
elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_au_fog_address"
elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_in_fog_address"
elif [ "${var_token#"$gem_prefix"}" != "${var_token}" ] || [ "${var_token#"$gem_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_gem_fog_address"
else

34
nodes/orchestration/package/orchestration_package.sh
View File

@ -43,6 +43,8 @@ DEFAULT_SETTINGS_PATH="${CONF_PATH}/settings.json"
var_default_fog_address="https://i2-agents.cloud.ngen.checkpoint.com/"
var_default_gem_fog_address="https://inext-agents.cloud.ngen.checkpoint.com"
var_default_us_fog_address="https://inext-agents-us.cloud.ngen.checkpoint.com"
var_default_au_fog_address="https://inext-agents-aus1.cloud.ngen.checkpoint.com"
var_default_in_fog_address="https://inext-agents-ind1.cloud.ngen.checkpoint.com"
var_fog_address=
var_certs_dir=
var_public_key=
@ -330,9 +332,17 @@ if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then
gem_prefix_uppercase="CP-"
us_prefix="cp-us-"
us_prefix_uppercase="CP-US-"
au_prefix="cp-au-"
au_prefix_uppercase="CP-AU-"
in_prefix="cp-in-"
in_prefix_uppercase="CP-IN-"
if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_us_fog_address"
elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_au_fog_address"
elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_in_fog_address"
elif [ "${var_token#"$gem_prefix"}" != "${var_token}" ] || [ "${var_token#"$gem_prefix_uppercase"}" != "${var_token}" ]; then
var_fog_address="$var_default_gem_fog_address"
else
@ -567,7 +577,7 @@ install_cp_nano_ctl()
if [ -f ${FILESYSTEM_PATH}/${CONF_PATH}/CP_NANO_AGENT_CTL ]; then
cp_exec "rm -rf ${FILESYSTEM_PATH}/${CONF_PATH}/$CP_NANO_AGENT_CTL"
fi
if [ -f ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${CP_NANO_AGENT_CTL_DEPRECATED} ]; then
cp_exec "rm -f ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${CP_NANO_AGENT_CTL_DEPRECATED} $USR_SBIN_PATH/${CP_NANO_CTL_DEPRECATED}"
fi
@ -602,7 +612,7 @@ set_conf_temp_location()
if ! cat ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} | grep -q "\"orchestration\":"; then
sed -i -e "1 s/{/{\n\"orchestration\": {\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}]},/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE}
else
sed -i -e "/\"orchestration\"/ s/\"orchestration\".*:.*{/\"orchestration\":{\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}],/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE}
sed -i -e "/\"orchestration\"/ s/\"orchestration\".*:.*{/\"orchestration\":{\"Default file download path\": [{\"value\":\""${escaped_temp_location}"\"}],/" ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE}
fi
fi
}
@ -753,6 +763,16 @@ install_public_key()
fi
}
uninstall_messaging_proxy_if_needed()
{
messaging_exec_path="${FILESYSTEM_PATH}/packages/messagingProxy/messagingProxy"
if [ -f ${messaging_exec_path} ]; then
chmod +x ${messaging_exec_path}
${messaging_exec_path} --uninstall
rm -rf ${FILESYSTEM_PATH}/packages/messagingProxy
fi
}
install_orchestration()
{
INSTALLATION_TIME=$(date)
@ -837,6 +857,8 @@ install_orchestration()
install_watchdog
cp_print "Upgrade to latest"
uninstall_messaging_proxy_if_needed
${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration "$var_arch_flag" > /dev/null 2>&1
if [ "$IS_K8S_ENV" = "true" ]; then
${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh
@ -1043,19 +1065,19 @@ run_pre_install_test()
run_post_install_test()
{
if [ $var_is_alpine = false ]; then
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so ]; then
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_chrono.so* ]; then
cp_print "Error, libboost_chrono .so file is missing" ${FORCE_STDOUT}
exit 1
fi
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so ]; then
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_context.so* ]; then
cp_print "Error, libboost_context .so file is missing" ${FORCE_STDOUT}
exit 1
fi
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so ]; then
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_system.so* ]; then
cp_print "Error, libboost_system .so file is missing" ${FORCE_STDOUT}
exit 1
fi
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so ]; then
if [ ! -f ${USR_LIB_PATH}/cpnano/libboost_thread.so* ]; then
cp_print "Error, libboost_thread .so file is missing" ${FORCE_STDOUT}
exit 1
fi