github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

June 27th update

Browse Source
This commit is contained in:
Ned Wright
2024-06-27 11:19:35 +00:00
parent 81b1aec487
commit 78b114a274
81 changed files with 1783 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/agent_core_utilities/agent_core_utilities.cc
View File

@@ -175,7 +175,7 @@ copyFile(const string &src, const string &dest, bool overide_if_exists, mode_t p
struct stat stat_buf;
int source_fd = open(src.c_str(), O_RDONLY);
fstat(source_fd, &stat_buf);
int dest_fd = open(dest.c_str(), O_WRONLY | O_CREAT, permission);
int dest_fd = open(dest.c_str(), O_WRONLY | O_CREAT | O_TRUNC, permission);
int bytes_copied = 1;
while (bytes_copied > 0) {
static const int buf_size = 4096*1000;

13
core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc
View File

@@ -37,8 +37,11 @@ TEST_F(AgentCoreUtilUT, filesTest)
EXPECT_FALSE(NGEN::Filesystem::exists("/i/am/not/a/real/path"));
const vector<string> lines{"i am a line in the text file", "i am iron man"};
const vector<string> lines_b{"i am a line 2 in the text file", "i am iron man 2", "hello again"};
CPTestTempfile test_file(lines);
CPTestTempfile test_file_b(lines_b);
ASSERT_TRUE(NGEN::Filesystem::exists(test_file.fname));
ASSERT_TRUE(NGEN::Filesystem::exists(test_file_b.fname));
string output_orig = test_file.readFile();
string new_path = test_file.fname + ".new";
@@ -46,6 +49,7 @@ TEST_F(AgentCoreUtilUT, filesTest)
ASSERT_TRUE(NGEN::Filesystem::exists(new_path));
ASSERT_FALSE(NGEN::Filesystem::copyFile(test_file.fname, new_path, false));
ASSERT_TRUE(NGEN::Filesystem::copyFile(test_file.fname, new_path, true));
ASSERT_TRUE(NGEN::Filesystem::copyFile(test_file.fname, test_file_b.fname, true));
string output_new;
{
ifstream new_file_stream(new_path);
@@ -55,11 +59,20 @@ TEST_F(AgentCoreUtilUT, filesTest)
output_new = buffer.str();
}
string output_test_b;
ifstream new_file_stream(test_file_b.fname);
ASSERT_TRUE(new_file_stream.good());
stringstream buffer;
buffer << new_file_stream.rdbuf();
output_test_b = buffer.str();
EXPECT_EQ(output_orig, output_new);
EXPECT_EQ(output_orig, output_test_b);
EXPECT_THAT(output_new, HasSubstr("i am a line in the text file"));
EXPECT_THAT(output_new, HasSubstr("i am iron man"));
EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file.fname));
EXPECT_TRUE(NGEN::Filesystem::deleteFile(new_path));
EXPECT_TRUE(NGEN::Filesystem::deleteFile(test_file_b.fname));
EXPECT_FALSE(NGEN::Filesystem::exists(test_file.fname));
EXPECT_FALSE(NGEN::Filesystem::exists(new_path));
}

14
core/config/config.cc
View File

@@ -348,14 +348,12 @@ ConfigComponent::Impl::init()
if (!Singleton::exists<I_MainLoop>()) return;
auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();
if (executable_name != "cp-nano-orchestration") {
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
[this] () { periodicRegistrationRefresh(); },
"Configuration update registration",
false
);
}
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
[this] () { periodicRegistrationRefresh(); },
"Configuration update registration",
false
);
}
static bool

93
core/debug_is/debug.cc
View File

@@ -234,6 +234,55 @@ public:
static DebugConfiguration default_config;
class ChangeDebugConfiguration : public ServerRest
{
public:
void
doCall() override
{
string debug_config_file_name = Debug::getExecutableName();
if (debug_config_file_name == "") {
output = "Error. Cannot get debug config file path";
return;
}
string debug_config_file_path = getConfigurationWithDefault<string>(
getFilesystemPathConfig() + "/conf/" + debug_config_file_name + "-debug-conf.json",
"Debug I/S",
"Debug conf file path"
);
try {
ifstream input_stream(debug_config_file_path);
if (!input_stream) {
output = "Error. Cannot open the debug conf file: " + debug_config_file_path;
return;
}
cereal::JSONInputArchive ar(input_stream);
Debug::prepareConfig();
vector<DebugConfiguration> debug_config;
try {
ar(cereal::make_nvp("Debug", debug_config));
} catch (cereal::Exception &e) {
output = "Error. Failed loading debug conf file, error: " + string(e.what());
Debug::abortConfig();
return;
}
input_stream.close();
setConfiguration(debug_config[0], "Debug");
Debug::commitConfig();
output = "New debug configuration set succesfully";
} catch (ifstream::failure &f) {
output =
"Error. Cannot open the debug conf file " +
debug_config_file_path +
", error: " +
string(f.what());
}
}
private:
S2C_PARAM(string, output);
};
// LCOV_EXCL_START - function is covered in unit-test, but not detected bt gcov
Debug::Debug(
const string &file_name,
@@ -446,6 +495,7 @@ Debug::preload()
{
registerExpectedConfiguration<DebugConfiguration>("Debug");
registerExpectedConfiguration<string>("Debug I/S", "Fog Debug URI");
registerExpectedConfiguration<string>("Debug I/S", "Debug conf file path");
registerExpectedConfiguration<bool>("Debug I/S", "Enable bulk of debugs");
registerExpectedConfiguration<uint>("Debug I/S", "Debug bulk size");
registerExpectedConfiguration<uint>("Debug I/S", "Debug bulk sending interval in msec");
@@ -467,22 +517,18 @@ Debug::init()
mainloop = Singleton::Consume<I_MainLoop>::by<Debug>();
env = Singleton::Consume<I_Environment>::by<Debug>();
auto executable = env->get<string>("Executable Name");
if (executable.ok() && *executable != "") {
string default_debug_output_file_path = *executable;
auto file_path_end = default_debug_output_file_path.find_last_of("/");
if (file_path_end != string::npos) {
default_debug_file_stream_path = default_debug_output_file_path.substr(file_path_end + 1);
}
auto file_sufix_start = default_debug_file_stream_path.find_first_of(".");
if (file_sufix_start != string::npos) {
default_debug_file_stream_path = default_debug_file_stream_path.substr(0, file_sufix_start);
}
default_debug_file_stream_path = getExecutableName();
if (default_debug_file_stream_path != "") {
string log_files_prefix = getLogFilesPathConfig();
default_debug_file_stream_path = log_files_prefix + "/nano_agent/" + default_debug_file_stream_path + ".dbg";
}
if (Singleton::exists<I_RestApi>()) {
Singleton::Consume<I_RestApi>::by<Debug>()->addRestCall<ChangeDebugConfiguration>(
RestAction::SET,
"change-debug-config"
);
}
}
void
@@ -706,6 +752,27 @@ Debug::findDebugFilePrefix(const string &file_name)
return "";
}
string
Debug::getExecutableName()
{
auto executable = env->get<string>("Executable Name");
if (!executable.ok() || *executable == "") {
return "";
}
string executable_name = *executable;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
return executable_name;
}
void
Debug::addActiveStream(const string &name)
{

1
core/debug_is/debug_ex.h
View File

@@ -21,6 +21,7 @@
#include "report/report.h"
#include "i_agent_details.h"
#include "i_environment.h"
#include "i_rest_api.h"
#include "i_mainloop.h"
#include "report/report_bulks.h"

13
core/debug_is/debug_is_ut/debug-ut-debug-conf.json Normal file
View File

@@ -0,0 +1,13 @@
{
"Debug": [
{
"Streams": [
{
"D_ALL": "Info",
"D_FW": "Debug",
"Output": "/var/log/nano_agent/cp-nano-orchestration.dbg"
}
]
}
]
}

59
core/debug_is/debug_is_ut/debug_ut.cc
View File

@@ -14,6 +14,7 @@
#include "mock/mock_messaging.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_environment.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_instance_awareness.h"
using namespace std;
@@ -551,7 +552,6 @@ public:
Debug::setNewDefaultStdout(&capture_debug);
}
~DebugConfigTest()
{
loadConfiguration("");
@@ -583,10 +583,19 @@ public:
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
}
bool
setDebugConfig(const unique_ptr<RestInit> &p)
{
set_debug_config = p->getRest();
return true;
}
ConfigComponent conf;
::Environment env;
stringstream capture_debug;
StrictMock<MockAgentDetails> mock_agent_details;
NiceMock<MockRestApi> mock_rest;
unique_ptr<ServerRest> set_debug_config;
};
TEST_F(DebugConfigTest, basic_configuration)
@@ -786,6 +795,54 @@ TEST_F(DebugConfigTest, fail_configuration)
EXPECT_FALSE(loadConfiguration(debug_config));
}
TEST_F(DebugConfigTest, testSetConfig)
{
NiceMock<MockMainLoop> mock_mainloop;
StrictMock<MockTimeGet> mock_time;
Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::WARNING);
EXPECT_TRUE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::ERROR));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::INFO));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::DEBUG));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::TRACE));
EXPECT_CALL(mock_rest, mockRestCall(RestAction::SET, "change-debug-config", _))
.WillOnce(WithArg<2>(Invoke(this, &DebugConfigTest::setDebugConfig))
);
Maybe<I_MainLoop::RoutineID> error_id = genError("no id");
EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(error_id));
EXPECT_CALL(mock_time, getWalltimeStr()).WillRepeatedly(Return(string("")));
EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
env.preload();
Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "debug-ut");
env.init();
Debug::init();
setConfiguration(
cptestFnameInSrcDir(string("debug-ut-debug-conf.json")),
string("Debug I/S"),
string("Debug conf file path")
);
stringstream ss("{}");
Maybe<string> maybe_res = set_debug_config->performRestCall(ss);
ASSERT_TRUE(maybe_res.ok());
EXPECT_EQ(maybe_res.unpack(), "{\n \"output\": \"New debug configuration set succesfully\"\n}");
EXPECT_TRUE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::ERROR));
EXPECT_TRUE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::INFO));
EXPECT_TRUE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::DEBUG));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::TRACE));
Debug::setUnitTestFlag(D_FW, Debug::DebugLevel::WARNING); // Reset debug level so it won't effect other tests
EXPECT_TRUE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::ERROR));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::INFO));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::DEBUG));
EXPECT_FALSE(Debug::isFlagAtleastLevel(D_FW, Debug::DebugLevel::TRACE));
Debug::fini();
}
ACTION(InvokeMainLoopCB)
{
arg1();

44
core/environment/environment.cc
View File

@@ -58,7 +58,6 @@ public:
string getCurrentTrace() const override;
string getCurrentSpan() const override;
string getCurrentHeaders() override;
map<string, string> getCurrentHeadersMap() override;
void startNewTrace(bool new_span, const string &_trace_id) override;
@@ -239,34 +238,6 @@ Environment::Impl::getCurrentSpan() const
return "";
}
string
Environment::Impl::getCurrentHeaders()
{
string tracing_headers;
auto trace_id = getCurrentTrace();
if (!trace_id.empty()) {
tracing_headers += "X-Trace-Id: " + trace_id + "\r\n";
} else {
string correlation_id_string = "00000000-0000-0000-0000-000000000000";
try {
boost::uuids::random_generator uuid_random_gen;
correlation_id_string = boost::uuids::to_string(uuid_random_gen());
} catch (const boost::uuids::entropy_error &e) {
dbgTrace(D_ENVIRONMENT)
<< "Failed to generate random correlation id - entropy exception. Exception: "
<< e.what();
tracing_status = TracingStatus::DISABLED;
}
tracing_headers += "X-Trace-Id: " + correlation_id_string + "\r\n";
}
auto span_id = getCurrentSpan();
if (!span_id.empty()) {
tracing_headers += "X-Span-Id: " + span_id + "\r\n";
}
return tracing_headers;
}
map<string, string>
Environment::Impl::getCurrentHeadersMap()
{
@@ -292,6 +263,21 @@ Environment::Impl::getCurrentHeadersMap()
if (!span_id.empty()) {
tracing_headers["X-Span-Id"] = span_id;
}
auto exec_name = get<string>("Executable Name");
if (exec_name.ok() && *exec_name != "") {
string executable_name = *exec_name;
auto file_path_end = executable_name.find_last_of("/");
if (file_path_end != string::npos) {
executable_name = executable_name.substr(file_path_end + 1);
}
auto file_sufix_start = executable_name.find_first_of(".");
if (file_sufix_start != string::npos) {
executable_name = executable_name.substr(0, file_sufix_start);
}
tracing_headers["X-Calling-Service"] = executable_name;
}
return tracing_headers;
}

10
core/environment/environment_ut/tracing_ut.cc
View File

@@ -345,12 +345,16 @@ public:
TEST_F(TracingCompRoutinesTest, 2SpansDifFlow)
{
I_MainLoop::Routine routine = [&] () {
string service_name = "test-service-name";
i_env->registerValue("Executable Name", service_name);
i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9");
trace_id = i_env->getCurrentTrace();
span_id = i_env->getCurrentSpan();
string headers = i_env->getCurrentHeaders();
EXPECT_THAT(headers, HasSubstr("X-Trace-Id: " + trace_id));
EXPECT_THAT(headers, HasSubstr("X-Span-Id: " + span_id));
auto headers = i_env->getCurrentHeadersMap();
EXPECT_THAT(headers["X-Trace-Id"], trace_id);
EXPECT_THAT(headers["X-Span-Id"], span_id);
EXPECT_THAT(headers["X-Calling-Service"], service_name);
EXPECT_EQ(trace_id, "a687b388-1108-4083-9852-07c33b1074e9");
EXPECT_NE("", i_env->getCurrentSpan());

3
core/include/general/debug.h
View File

@@ -30,6 +30,7 @@ class I_Environment;
class I_InstanceAwareness;
class I_Encryptor;
class I_AgentDetails;
class I_RestApi;
class I_SignalHandler;
namespace Config { enum class Errors; }
@@ -44,6 +45,7 @@ class Debug
Singleton::Consume<I_Environment>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_SignalHandler>
{
public:
@@ -178,6 +180,7 @@ public:
static void setUnitTestFlag(DebugFlags flag, DebugLevel level);
static std::string findDebugFilePrefix(const std::string &file_name);
static std::string getExecutableName();
private:
template <typename T, typename... Args>

2
core/include/internal/messaging.h
View File

@@ -26,6 +26,7 @@
#include "i_time_get.h"
#include "i_encryptor.h"
#include "i_shell_cmd.h"
#include "i_rest_api.h"
#include "i_instance_awareness.h"
#include "config.h"
@@ -41,6 +42,7 @@ class Messaging
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_RestApi>,
Singleton::Consume<I_InstanceAwareness>
{
public:

1
core/include/services_sdk/interfaces/i_environment.h
View File

@@ -79,7 +79,6 @@ public:
virtual std::string getCurrentTrace() const = 0;
virtual std::string getCurrentSpan() const = 0;
virtual std::string getCurrentHeaders() = 0;
virtual std::map<std::string, std::string> getCurrentHeadersMap() = 0;
virtual void startNewTrace(bool new_span = true, const std::string &_trace_id = std::string()) = 0;
virtual void startNewSpan(

10
core/include/services_sdk/interfaces/i_intelligence_is_v2.h
View File

@@ -46,11 +46,17 @@ public:
const std::vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_bulk,
bool is_proxy,
const MessageMetadata &req_md
) const = 0;
virtual Maybe<Intelligence::Response>
getResponse(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md) const = 0;
getResponse(
const QueryRequest &query_request,
bool is_pretty,
bool is_proxy,
const MessageMetadata &req_md
) const = 0;
template<typename Data>
Maybe<std::vector<AssetReply<Data>>>
@@ -58,6 +64,7 @@ public:
QueryRequest &query_request,
bool ignore_in_progress = false,
bool is_pretty = true,
bool is_proxy = false,
MessageMetadata req_md = MessageMetadata("", 0)
);
@@ -66,6 +73,7 @@ public:
queryIntelligence(
std::vector<QueryRequest> &query_requests,
bool is_pretty = true,
bool is_proxy = false,
MessageMetadata req_md = MessageMetadata("", 0)
);

6
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h
View File

@@ -24,10 +24,11 @@ I_Intelligence_IS_V2::queryIntelligence(
QueryRequest &query_request,
bool ignore_in_progress,
bool is_pretty,
bool is_proxy,
MessageMetadata req_md
)
{
auto response = getResponse(query_request, is_pretty, req_md);
auto response = getResponse(query_request, is_pretty, is_proxy, req_md);
if (!response.ok()) return response.passErr();
auto serializable_response = response->getSerializableResponse<Data>();
@@ -49,10 +50,11 @@ Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
I_Intelligence_IS_V2::queryIntelligence(
std::vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_proxy,
MessageMetadata req_md
)
{
auto res = getResponse(query_requests, is_pretty, true, req_md);
auto res = getResponse(query_requests, is_pretty, true, is_proxy, req_md);
if (!res.ok()) return res.passErr();
return res->getBulkData<Data>();

1
core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h
View File

@@ -73,6 +73,7 @@ public:
SerializableQueryFilter(Condition condition_type, const std::string &key, const std::string &value);
SerializableQueryFilter(Condition condition_type, const std::string &key, const int64_t &value);
SerializableQueryFilter(Condition condition_type, const std::string &key, const std::vector<std::string> &value);
SerializableQueryFilter(const SerializableQueryCondition &condition);
void save(cereal::JSONOutputArchive &ar) const;

1
core/include/services_sdk/interfaces/mock/mock_environment.h
View File

@@ -28,7 +28,6 @@ public:
MOCK_CONST_METHOD0(getCurrentTrace, std::string());
MOCK_CONST_METHOD0(getCurrentSpan, std::string());
MOCK_METHOD0(getCurrentHeaders, std::string());
MOCK_METHOD0(getCurrentHeadersMap, std::map<std::string, std::string>());
MOCK_METHOD2(startNewTrace, void(bool, const std::string &));
MOCK_METHOD3(startNewSpan, void(Span::ContextType, const std::string &, const std::string &));

13
core/include/services_sdk/interfaces/mock/mock_intelligence.h
View File

@@ -26,21 +26,26 @@ public:
MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
MOCK_METHOD1(unregisterInvalidation, void(uint id));
MOCK_CONST_METHOD4(
MOCK_CONST_METHOD5(
getResponse,
Maybe<Response>(
const std::vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_bulk,
bool is_proxy,
const MessageMetadata &req_md
)
);
MOCK_CONST_METHOD3(
MOCK_CONST_METHOD4(
getResponse,
Maybe<Response>(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md)
Maybe<Response>(
const QueryRequest &query_request,
bool is_pretty,
bool is_proxy,
const MessageMetadata &req_md
)
);
MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void));
MOCK_CONST_METHOD1(getOfflineInfoString, Maybe<std::string>(const SerializableQueryFilter &query));
};
#endif // __MOCK_INTELLIGENCE_H__

2
core/include/services_sdk/resources/component_is/components_list_impl.h
View File

@@ -209,13 +209,13 @@ class ComponentListCore
ShellCmd,
GenericMetric,
Messaging,
MainloopComponent,
ConfigComponent,
InstanceAwareness,
IntelligenceComponentV2,
AgentDetails,
LoggingComp,
TimeProxyComponent,
MainloopComponent,
SignalHandler,
RestServer,
Encryptor,

1
core/include/services_sdk/resources/debug_flags.h
View File

@@ -142,6 +142,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_LOCAL_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_NGINX_POLICY, D_ORCHESTRATOR)
DEFINE_FLAG(D_SERVICE_CONTROLLER, D_ORCHESTRATOR)
DEFINE_FLAG(D_UPDATES_PROCESS_REPORTER, D_ORCHESTRATOR)
DEFINE_FLAG(D_GRADUAL_DEPLOYMENT, D_COMPONENT)
DEFINE_FLAG(D_SDWAN, D_COMPONENT)

7
core/include/services_sdk/resources/health_check_status/health_check_status.h
View File

@@ -89,11 +89,4 @@ private:
std::map<std::string, std::string> extended_status = {};
};
class HealthCheckStatusEvent : public Event<HealthCheckStatusEvent, HealthCheckStatusReply>
{
public:
HealthCheckStatusEvent() {}
~HealthCheckStatusEvent() {}
};
#endif // __HEALTH_CHECK_STATUS_H__

2
core/include/services_sdk/resources/report/report_enums.h
View File

@@ -69,6 +69,8 @@ enum class Tags {
NGINX_PROXY_MANAGER,
WEB_SERVER_APISIX,
DEPLOYMENT_DOCKER,
WEB_SERVER_SWAG,
WEB_SERVER_NGINX_UNIFIED,
COUNT
};

4
core/intelligence_is_v2/include/intelligence_request.h
View File

@@ -29,10 +29,11 @@ public:
const std::vector<QueryRequest> &queries,
bool is_pretty,
bool is_bulk,
bool _is_proxy,
const MessageMetadata &req_md
)
:
queries(queries), is_pretty(is_pretty), is_bulk(is_bulk), req_md(req_md)
queries(queries), is_pretty(is_pretty), is_bulk(is_bulk), is_proxy(_is_proxy), req_md(req_md)
{}
Maybe<void> checkAssetsLimit() const;
@@ -51,6 +52,7 @@ private:
const std::vector<QueryRequest> &queries;
bool is_pretty = true;
bool is_bulk = false;
bool is_proxy = false;
Maybe<std::string> response_from_fog = genError("Uninitialized");
const MessageMetadata &req_md;
};

12
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@@ -340,10 +340,11 @@ public:
const vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_bulk,
bool is_proxy,
const MessageMetadata &req_md
) const override
{
IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk, req_md);
IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk, is_proxy, req_md);
if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr();
if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr();
if (intelligence_req.isPagingActivated()) {
@@ -357,10 +358,15 @@ public:
}
Maybe<Intelligence::Response>
getResponse(const QueryRequest &query_request, bool is_pretty, const MessageMetadata &req_md) const override
getResponse(
const QueryRequest &query_request,
bool is_pretty,
bool is_proxy,
const MessageMetadata &req_md
) const override
{
vector<QueryRequest> queries = {query_request};
return getResponse(queries, is_pretty, false, req_md);
return getResponse(queries, is_pretty, false, is_proxy, req_md);
}
private:

6
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@@ -131,7 +131,7 @@ TEST_F(IntelligenceComponentMockTest, getResponseErrorTest)
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
Maybe<Intelligence::Response> res_error = genError("Test error");
EXPECT_CALL(intelligence_mock, getResponse(_, _, _)
EXPECT_CALL(intelligence_mock, getResponse(_, _, _, _)
).WillOnce(Return(res_error));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
@@ -185,7 +185,7 @@ TEST_F(IntelligenceComponentMockTest, getResponseTest)
Intelligence::Response response(response_str, 1, false);
EXPECT_CALL(intelligence_mock, getResponse(_, _, _)
EXPECT_CALL(intelligence_mock, getResponse(_, _, _, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
@@ -346,7 +346,7 @@ TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest)
);
Intelligence::Response response(response_str, 4, true);
EXPECT_CALL(intelligence_mock, getResponse(_, _, _, _)
EXPECT_CALL(intelligence_mock, getResponse(_, _, _, _, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(requests);

87
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
View File

@@ -24,9 +24,33 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, true, false, MessageMetadata("", 0));
Intelligence::IntelligenceRequest query(requests, true, false, false, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"proxyToCloud\": false\n"
" },\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing\"\n"
" }\n"
"}";
EXPECT_EQ(*query.genJson(), expected);
}
TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequestProxied) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, true, false, true, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"proxyToCloud\": true\n"
" },\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
@@ -42,9 +66,12 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false, MessageMetadata("", 0));
Intelligence::IntelligenceRequest query(requests, false, false, false, MessageMetadata("", 0));
std::string expected = "{"
"\"queryTypes\":{"
"\"proxyToCloud\":false"
"},"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
@@ -59,8 +86,11 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) {
QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false, MessageMetadata("", 0));
Intelligence::IntelligenceRequest query(requests, false, false, false, MessageMetadata("", 0));
std::string expected = "{"
"\"queryTypes\":{"
"\"proxyToCloud\":false"
"},"
"\"limit\":20,"
"\"fullResponse\":true,"
"\"query\":{"
@@ -76,9 +106,53 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
Intelligence::IntelligenceRequest query(requests, true, true, MessageMetadata("", 0));
Intelligence::IntelligenceRequest query(requests, true, true, false, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"proxyToCloud\": false\n"
" },\n"
" \"queries\": [\n"
" {\n"
" \"query\": {\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.phase\",\n"
" \"value\": \"testing\"\n"
" }\n"
" },\n"
" \"index\": 0\n"
" },\n"
" {\n"
" \"query\": {\n"
" \"limit\": 20,\n"
" \"fullResponse\": true,\n"
" \"query\": {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.height\",\n"
" \"value\": \"testing\"\n"
" }\n"
" },\n"
" \"index\": 1\n"
" }\n"
" ]\n"
"}";
EXPECT_EQ(*query.genJson(), expected);
}
TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequestsProxied) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
Intelligence::IntelligenceRequest query(requests, true, true, true, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"proxyToCloud\": true\n"
" },\n"
" \"queries\": [\n"
" {\n"
" \"query\": {\n"
@@ -114,9 +188,12 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
Intelligence::IntelligenceRequest query(requests, false, true, MessageMetadata("", 0));
Intelligence::IntelligenceRequest query(requests, false, true, false, MessageMetadata("", 0));
std::string expected = "{"
"\"queryTypes\":{"
"\"proxyToCloud\":false"
"},"
"\"queries\":[{"
"\"query\":{"
"\"limit\":20,"

56
core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
View File

@@ -705,3 +705,59 @@ TEST(QueryRequestTestV2, UninitializedObjectTypeTest)
EXPECT_THAT(debug_output.str(), HasSubstr(debug_str));
Debug::setNewDefaultStdout(&cout);
}
TEST(QueryRequestTestV2, Bug40968)
{
QueryRequest request(Intelligence_IS_V2::Condition::EQUALS, "field1", "123", false);
QueryRequest filter1(Intelligence_IS_V2::Condition::EQUALS, "field2", "123", false);
request = request || filter1;
QueryRequest filter2(Intelligence_IS_V2::Condition::NOT_EQUALS, "field3", "123", false);
request = request && filter2;
QueryRequest filter3(Intelligence_IS_V2::Condition::EQUALS, "field3", "234", false);
request = request && filter3;
stringstream out2;
{
cereal::JSONOutputArchive out_ar2(out2);
request.saveToJson(out_ar2);
}
string req =
"{\n"
" \"limit\": 20,\n"
" \"fullResponse\": false,\n"
" \"query\": {\n"
" \"operator\": \"and\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"or\",\n"
" \"operands\": [\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.field1\",\n"
" \"value\": \"123\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.field2\",\n"
" \"value\": \"123\"\n"
" }\n"
" ]\n"
" },\n"
" {\n"
" \"operator\": \"notEquals\",\n"
" \"key\": \"mainAttributes.field3\",\n"
" \"value\": \"123\"\n"
" },\n"
" {\n"
" \"operator\": \"equals\",\n"
" \"key\": \"mainAttributes.field3\",\n"
" \"value\": \"234\"\n"
" }\n"
" ]\n"
" }\n"
"}";
EXPECT_EQ(out2.str(), req);
}

6
core/intelligence_is_v2/intelligence_request.cc
View File

@@ -89,6 +89,12 @@ IntelligenceRequest::genJson() const
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(json_stream);
out_ar.setNextName("queryTypes");
out_ar.startNode();
out_ar(cereal::make_nvp("proxyToCloud", is_proxy));
out_ar.finishNode();
if (isBulk()) {
out_ar.setNextName("queries");
out_ar.startNode();

56
core/intelligence_is_v2/query_filter_v2.cc
View File

@@ -65,6 +65,11 @@ SerializableQueryFilter::SerializableQueryFilter(
condition_operands.emplace_back(condition_type, key, value);
}
SerializableQueryFilter::SerializableQueryFilter(const SerializableQueryCondition &condition)
{
condition_operands.push_back(condition);
}
SerializableQueryFilter::SerializableQueryFilter(
Condition condition_type,
@@ -159,31 +164,38 @@ SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query
query_filter_res.operator_type = oper;
if (isOperatorComp(oper) && other_query.isOperatorComp(oper)) {
size_t queries_size = queries_operands.size() + other_query.queries_operands.size();
size_t conditions_size = condition_operands.size() + other_query.condition_operands.size();
query_filter_res.queries_operands.reserve(queries_size);
query_filter_res.condition_operands.reserve(conditions_size);
for (const auto &subquery : queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
for (const auto &subquery : other_query.queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : other_query.condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
} else {
if (!isOperatorComp(oper) || !other_query.isOperatorComp(oper)) {
query_filter_res.queries_operands.reserve(2);
query_filter_res.queries_operands.push_back(*this);
query_filter_res.queries_operands.push_back(other_query);
return query_filter_res;
}
if (!condition_operands.empty() && !other_query.condition_operands.empty()) {
query_filter_res.condition_operands.reserve(queries_operands.size() + other_query.queries_operands.size());
for (const auto &condition : condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
for (const auto &condition : other_query.condition_operands) {
query_filter_res.condition_operands.push_back(condition);
}
return query_filter_res;
}
size_t queries_size = queries_operands.size() + other_query.queries_operands.size();
size_t conditions_size = condition_operands.size() + other_query.condition_operands.size();
query_filter_res.queries_operands.reserve(queries_size + conditions_size);
for (const auto &subquery : queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : condition_operands) {
query_filter_res.queries_operands.emplace_back(condition);
}
for (const auto &subquery : other_query.queries_operands) {
query_filter_res.queries_operands.push_back(subquery);
}
for (const auto &condition : other_query.condition_operands) {
query_filter_res.queries_operands.emplace_back(condition);
}
return query_filter_res;

42
core/messaging/messaging_comp/messaging_comp.cc
View File

@@ -18,6 +18,7 @@
#include "agent_core_utilities.h"
#include "connection_comp.h"
#include "rest.h"
#include "debug.h"
#include "messaging_buffer.h"
@@ -25,6 +26,40 @@ using namespace std;
USE_DEBUG_FLAG(D_MESSAGING);
class FogConnectionChecker : public ServerRest
{
public:
void
doCall() override
{
dbgTrace(D_MESSAGING) << "Checking connection to the FOG";
auto response = Singleton::Consume<I_Messaging>::from<Messaging>()->sendSyncMessage(
HTTPMethod::GET,
"/access-manager/health/live",
string("")
);
if (!response.ok()) {
dbgTrace(D_MESSAGING) << "Failed to check connection to the FOG";
connected_to_fog = false;
error = response.getErr().toString();
return;
}
if (response.unpack().getHTTPStatusCode() == HTTPStatusCode::HTTP_OK) {
dbgTrace(D_MESSAGING) << "Connected to the FOG";
connected_to_fog = true;
error = "";
} else {
dbgTrace(D_MESSAGING) << "No connection to the FOG";
connected_to_fog = false;
error = response.unpack().toString();
}
}
private:
S2C_PARAM(bool, connected_to_fog);
S2C_PARAM(string, error);
};
void
MessagingComp::init()
{
@@ -42,6 +77,13 @@ MessagingComp::init()
"message",
"Buffer Failed Requests"
);
if (Singleton::exists<I_RestApi>()) {
Singleton::Consume<I_RestApi>::by<Messaging>()->addRestCall<FogConnectionChecker>(
RestAction::SHOW,
"check-fog-connection"
);
}
}
static bool

51
core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
View File

@@ -15,6 +15,8 @@
#include "mocks/mock_messaging_connection.h"
#include "rest.h"
#include "rest_server.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "dummy_socket.h"
using namespace std;
@@ -26,12 +28,6 @@ operator<<(ostream &os, const Maybe<BufferedMessage> &)
return os;
}
static std::ostream &
operator<<(std::ostream &os, const HTTPResponse &)
{
return os;
}
static std::ostream &
operator<<(std::ostream &os, const HTTPStatusCode &)
{
@@ -51,6 +47,9 @@ public:
{
Debug::setUnitTestFlag(D_MESSAGING, Debug::DebugLevel::TRACE);
EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0)));
EXPECT_CALL(mock_rest, mockRestCall(RestAction::SHOW, "check-fog-connection", _))
.WillOnce(WithArg<2>(Invoke(this, &TestMessagingComp::showFogConnection))
);
ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe<string>(fog_addr)));
ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe<uint16_t>(fog_port)));
@@ -73,6 +72,13 @@ public:
EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred")));
}
bool
showFogConnection(const unique_ptr<RestInit> &p)
{
show_fog_connection = p->getRest();
return true;
}
const string fog_addr = "127.0.0.1";
int fog_port = 8080;
CPTestTempfile agent_details_file;
@@ -85,16 +91,37 @@ public:
NiceMock<MockTimeGet> mock_time_get;
NiceMock<MockAgentDetails> mock_agent_details;
NiceMock<MockProxyConfiguration> mock_proxy_conf;
NiceMock<MockRestApi> mock_rest;
NiceMock<MockMessaging> mock_message;
unique_ptr<ServerRest> show_fog_connection;
DummySocket dummy_socket;
};
TEST_F(TestMessagingComp, testInitComp)
TEST_F(TestMessagingComp, testCheckFogConnectivity)
{
EXPECT_CALL(
mock_mainloop, addRecurringRoutine(I_MainLoop::RoutineType::Timer, _, _, "Delete expired cache entries", _)
)
.WillOnce(Return(0));
messaging_comp.init();
setAgentDetails();
HTTPResponse res(
HTTPStatusCode::HTTP_OK,
string(
"{"
" \"up\": true,"
" \"timestamp\":\"\""
"}"
)
);
EXPECT_CALL(mock_message, sendSyncMessage(HTTPMethod::GET, "/access-manager/health/live", "", _, _))
.WillOnce(Return(res));
stringstream ss("{}");
Maybe<string> maybe_res = show_fog_connection->performRestCall(ss);
EXPECT_TRUE(maybe_res.ok());
EXPECT_EQ(maybe_res.unpack(),
"{\n"
" \"connected_to_fog\": true,\n"
" \"error\": \"\"\n"
"}"
);
}
TEST_F(TestMessagingComp, testSendSyncMessage)

8
core/report/tag_and_enum_management.cc
View File

@@ -111,7 +111,9 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
{"Playground", ReportIS::Tags::PLAYGROUND},
{"Nginx Proxy Manager", ReportIS::Tags::NGINX_PROXY_MANAGER},
{"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX},
{"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER}
{"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER},
{"SWAG Server", ReportIS::Tags::WEB_SERVER_SWAG},
{"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED}
};
auto report_is_tag = strings_to_tags.find(tag);
@@ -320,7 +322,9 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
"apiDiscoveryCloudMessaging",
"Nginx Proxy Manager",
"APISIX Server",
"Docker Deployment"
"Docker Deployment",
"SWAG Server",
"NGINX Unified Server"
};
EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {

2
core/version/CMakeLists.txt
View File

@@ -2,7 +2,7 @@ set(VERSION_VARS_H_FILE ${CMAKE_CURRENT_BINARY_DIR}/version_vars.h)
set(BUILD_SCRIPT build_version_vars_h.py)
add_custom_command(
OUTPUT ${VERSION_VARS_H_FILE}
COMMAND CI_PIPELINE_ID=00000001 CI_BUILD_REF_NAME=open-source python3 ${BUILD_SCRIPT} "userspace" > ${VERSION_VARS_H_FILE}
COMMAND CI_PIPELINE_ID=00000001 CI_COMMIT_REF_NAME=open-source python3 ${BUILD_SCRIPT} "userspace" > ${VERSION_VARS_H_FILE}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
DEPENDS ${BUILD_SCRIPT}
)

2
core/version/build_version_vars_h.py
View File

@@ -15,7 +15,7 @@ timestamp = "%s%s" % (now.replace(microsecond=0).isoformat(), time.strftime("%z"
version_prefix = ""
full_version = "private"
branch = os.getenv("CI_BUILD_REF_NAME")
branch = os.getenv("CI_COMMIT_REF_NAME")
if branch is None:
branch = "private"