github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Nov_12_2023-Dev

Browse Source
This commit is contained in:
Ned Wright
2023-11-12 18:50:17 +00:00
parent 0869b8f24d
commit 3061342b45
114 changed files with 3627 additions and 1305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
core/agent_core_utilities/agent_core_utilities.cc
View File

@@ -68,6 +68,29 @@ isDirectory(const string &path)
return false;
}
Maybe<vector<string>>
getDirectoryFiles(const string &path)
{
if (!isDirectory(path)) return genError("Path: " + path + " is not a directory");
struct dirent *entry = nullptr;
DIR *directory = opendir(path.c_str());
if (!directory) {
dbgWarning(D_INFRA_UTILS) << "Fail to open directory. Path: " << path << ", Errno: " << errno;
return genError("Failed to open directory: " + path);
}
vector<string> files;
while ((entry = readdir(directory))) {
if (entry->d_type == DT_REG) files.push_back(entry->d_name);
}
closedir(directory);
return files;
}
bool
makeDir(const string &path, mode_t permission)
{
@@ -257,13 +280,17 @@ regexMatch(const char *file, int line, const char *sample, cmatch &match, const
try {
return regex_match(sample, match, regex);
} catch (const runtime_error &err) {
uint sample_len = strlen(sample);
dbgError(D_INFRA_UTILS)
<< "FAILURE during regex_match @ "
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample_len
<< " sample='"
<< string(sample, min(100u, sample_len))
<< "', pattern='"
<< regex.str()
<< "': "
<< err.what();
@@ -282,8 +309,11 @@ regexMatch(const char *file, int line, const string &sample, smatch &match, cons
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample.size()
<< " sample='"
<< sample.substr(0, 100)
<< "', pattern='"
<< regex.str()
<< "': "
<< err.what();
@@ -302,8 +332,11 @@ regexMatch(const char *file, int line, const string &sample, const regex &regex)
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample.size()
<< " sample='"
<< sample.substr(0, 100)
<< "', pattern='"
<< regex.str()
<< "': "
<< err.what();
@@ -322,8 +355,11 @@ regexMatch(const char *file, int line, string &sample, const regex &regex)
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample.size()
<< " sample='"
<< sample.substr(0, 100)
<< "', pattern='"
<< regex.str()
<< "': "
<< err.what();
@@ -342,8 +378,11 @@ regexSearch(const char *file, int line, const string &sample, smatch &match, con
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample.size()
<< " sample='"
<< sample.substr(0, 100)
<< "', pattern='"
<< regex.str()
<< "': "
<< err.what();
@@ -362,8 +401,11 @@ regexReplace(const char *file, int line, const string &sample, const regex &rege
<< file
<< ":"
<< line
<< "; sample='"
<< sample << "', pattern='"
<< "; sample size: "
<< sample.size()
<< " sample='"
<< sample.substr(0, 100)
<< "', pattern='"
<< regex.str()
<< "', replace='"
<< replace

25
core/config/config.cc
View File

@@ -52,12 +52,15 @@ public:
class LoadNewConfigurationStatus : public ClientRest
{
public:
LoadNewConfigurationStatus(uint _id, bool _error, bool end) : id(_id), error(_error), finished(end) {}
LoadNewConfigurationStatus(uint _id, string _service_name, bool _error, bool end)
:
id(_id), service_name(_service_name), error(_error), finished(end) {}
void setError(const string &error) { error_message = error; }
private:
C2S_PARAM(int, id);
C2S_PARAM(string, service_name);
C2S_PARAM(bool, error);
C2S_PARAM(bool, finished);
C2S_OPTIONAL_PARAM(string, error_message);
@@ -133,7 +136,7 @@ public:
void registerExpectedSetting(unique_ptr<GenericConfig<false>> &&config) override;
bool loadConfiguration(istream &json_contents) override;
bool loadConfiguration(istream &json_contents, const string &path) override;
bool loadConfiguration(const vector<string> &configuration_flags) override;
AsyncLoadConfigStatus reloadConfiguration(const string &version, bool is_async, uint id) override;
bool saveConfiguration(ostream &) const override { return false; }
@@ -565,13 +568,13 @@ ConfigComponent::Impl::registerExpectedSetting(unique_ptr<GenericConfig<false>>
}
bool
ConfigComponent::Impl::loadConfiguration(istream &stream)
ConfigComponent::Impl::loadConfiguration(istream &stream, const string &path)
{
vector<shared_ptr<JSONInputArchive>> archive;
try {
archive.emplace_back(make_shared<JSONInputArchive>(stream));
} catch (const cereal::Exception &e) {
dbgError(D_CONFIG) << "Failed to load stream: " << e.what();
dbgError(D_CONFIG) << "Failed to serialize stream. Path: " << path << ", Error: " << e.what();
return false;
}
return loadConfiguration(archive, false);
@@ -872,7 +875,12 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
for (const auto &file : files) {
if (file.second->is_open()) {
dbgTrace(D_CONFIG) << "Succesfully opened configuration file. File: " << file.first;
archives.push_back(make_shared<JSONInputArchive>(*file.second));
try {
archives.push_back(make_shared<JSONInputArchive>(*file.second));
} catch (const cereal::Exception &e) {
dbgError(D_CONFIG) << "Failed in file serialization. Path: " << file.first << ", Error: " << e.what();
return false;
}
} else {
dbgTrace(D_CONFIG) << "Could not open configuration file. Path: " << file.first;
}
@@ -904,8 +912,9 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version
{
dbgFlow(D_CONFIG) << "Running reloadConfigurationContinuesWrapper. Version: " << version << ", Id: " << id;
auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();
LoadNewConfigurationStatus in_progress(id, false, false);
auto maybe_service_name = Singleton::Consume<I_Environment>::by<ConfigComponent>()->get<string>("Service Name");
string service_name = maybe_service_name.ok() ? maybe_service_name.unpack() : "serviceNameNotRegistered";
LoadNewConfigurationStatus in_progress(id, service_name, false, false);
auto routine_id = mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::Timer,
std::chrono::seconds(30),
@@ -916,7 +925,7 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version
bool res = reloadConfigurationImpl(version, true);
mainloop->stop(routine_id);
LoadNewConfigurationStatus finished(id, !res, true);
LoadNewConfigurationStatus finished(id, service_name, !res, true);
if (!res) finished.setError("Failed to reload configuration");
sendOrchestatorReloadStatusMsg(finished);

2
core/include/attachments/nginx_attachment_common.h
View File

@@ -128,6 +128,8 @@ typedef enum ngx_http_plugin_metric_type
AVERAGE_RSS_MEMORY_USAGE,
MAX_VM_MEMORY_USAGE,
MAX_RSS_MEMORY_USAGE,
REQUEST_OVERALL_SIZE_COUNT,
RESPONSE_OVERALL_SIZE_COUNT,
METRIC_TYPES_COUNT
} ngx_http_plugin_metric_type_e;

2
core/include/services_sdk/resources/config/i_config.h
View File

@@ -88,7 +88,7 @@ public:
// TODO: merge both loadConfiguration functions to one with vector of streams input when moving to c++17
// (c++ < 17 does not support copy of streams and thus it cannot be part of any container)
virtual bool loadConfiguration(istream &json_contents) = 0;
virtual bool loadConfiguration(istream &json_contents, const string &path = "") = 0;
virtual bool loadConfiguration(const vector<string> &configuration_flags) = 0;
virtual AsyncLoadConfigStatus

3
core/include/services_sdk/resources/debug_flags.h
View File

@@ -60,6 +60,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_STREAMING_DATA, D_STREAMING)
DEFINE_FLAG(D_CHECKSUM, D_STREAMING)
DEFINE_FLAG(D_WAAP, D_COMPONENT)
DEFINE_FLAG(D_OA_SCHEMA_UPDATER, D_WAAP)
DEFINE_FLAG(D_WAAP_API, D_WAAP)
DEFINE_FLAG(D_WAAP_AUTOMATION, D_WAAP)
DEFINE_FLAG(D_WAAP_REGEX, D_WAAP)
@@ -76,6 +77,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP)
DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER, D_WAAP)
DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_HTML, D_WAAP_PARSER)
@@ -91,6 +93,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
DEFINE_FLAG(D_WAAP_PARSER_URLENCODE, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP)
DEFINE_FLAG(D_IPS, D_COMPONENT)

7
core/include/services_sdk/resources/report/report_enums.h
View File

@@ -65,6 +65,7 @@ enum class Tags {
HORIZON_TELEMETRY_METRICS,
CROWDSEC,
PLAYGROUND,
API_DISCOVERY,
COUNT
};
@@ -152,8 +153,10 @@ enum class IssuingEngine {
IOT_NEXT,
SDWAN,
FILE_UPLOAD,
IDA_NEXT,
HORIZON_TELEMETRY_METRICS
IDA_NEXT_BLADE_REGISTRATION,
IDA_NEXT_CLIENT_IP_NOTIFY,
HORIZON_TELEMETRY_METRICS,
API_DISCOVERY
};
} // namespace ReportIS

4
core/include/services_sdk/utilities/agent_core_utilities.h
View File

@@ -18,6 +18,8 @@
#include <string>
#include <boost/regex.hpp>
#include "maybe_res.h"
namespace NGEN
{
@@ -26,7 +28,7 @@ namespace Filesystem
bool exists(const std::string &path);
bool isDirectory(const std::string &path);
Maybe<std::vector<std::string>> getDirectoryFiles(const std::string &path);
bool makeDir(const std::string &path, mode_t permission = S_IRWXU);
bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU);
bool deleteDirectory(const std::string &path, bool delete_content = false);

10
core/report/tag_and_enum_management.cc
View File

@@ -107,9 +107,10 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
{"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL},
{"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS},
{"Crowdsec", ReportIS::Tags::CROWDSEC},
{"apiDiscoveryCloudMessaging", ReportIS::Tags::API_DISCOVERY},
{"Playground", ReportIS::Tags::PLAYGROUND}
};
auto report_is_tag = strings_to_tags.find(tag);
if (report_is_tag != strings_to_tags.end()) return report_is_tag->second;
return genError("illegal tag: " + tag);
@@ -267,7 +268,9 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine)
case IssuingEngine::IOT_NEXT: return "iotNext";
case IssuingEngine::SDWAN: return "sdwanGwSharing";
case IssuingEngine::FILE_UPLOAD: return "fileUpload";
case IssuingEngine::IDA_NEXT: return "quantumMetaNotifyIdn";
case IssuingEngine::IDA_NEXT_BLADE_REGISTRATION: return "quantumMetaNotifyIdn";
case IssuingEngine::IDA_NEXT_CLIENT_IP_NOTIFY: return "quantumIPNotifyIdn";
case IssuingEngine::API_DISCOVERY: return "apiDiscoveryCloudMessaging";
case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics";
}
@@ -310,7 +313,8 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
"Layer 7 Access Control",
"Horizon Telemetry Metrics",
"Crowdsec",
"Playground"
"Playground",
"apiDiscoveryCloudMessaging"
};
EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {

40
core/version/version_ut/version_ut.cc
View File

@@ -17,49 +17,9 @@ TEST(Version, format)
ContainsRegex("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[-+][0-9]{4}")
);
// "Build 123" or "GitID 7d67870"
EXPECT_THAT(Version::getID(), ContainsRegex("([0-9]+)|[0-9]{4}.([0-9]+)"));
// get() return all parts of information, timestamp and id.
EXPECT_THAT(Version::get(), ContainsRegex("([0-9]+)|[0-9]{4}.([0-9]+)"));
EXPECT_THAT(Version::get(), ContainsRegex("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[-+][0-9]{4}"));
}
TEST(Version, getVerPrefix)
{
EXPECT_EQ("1.", Version::getVerPrefix());
}
TEST(Version, getUser)
{
if (Version::isPublic()) {
// public builds call this function but don't use the return value
// ut will do the same, as the user name is not accessible in public builds.
auto user = Version::getUser();
const char* buffer = getenv("CI_BUILD_REF_NAME");
ASSERT_FALSE(!buffer);
EXPECT_THAT(Version::getBranch(), AnyOf(buffer, StartsWith("pipeline")));
} else {
// Version::getUser is define by the python function: getpass.getuser().
// The getuser() function displays the login name of the user.
// This function checks the environment variables LOGNAME, USER, LNAME and USERNAME, in order,
// and returns the value of the first non-empty string.
const char* buffer = getenv("LOGNAME");
if (!buffer) {
buffer = getenv("USER");
if (!buffer) {
buffer = getenv("LNAME");
if (!buffer) {
buffer = getenv("USERNAME");
}
}
}
ASSERT_FALSE(!buffer);
EXPECT_EQ(buffer, Version::getUser());
EXPECT_EQ(Version::getBranch(), "private");
}
}
unique_ptr<ServerRest> show_version;
bool showVersion(const unique_ptr<RestInit> &p) { show_version = p->getRest(); return true; }