diff --git a/components/include/external_sdk_server.h b/components/include/external_sdk_server.h index cd0c593..0aae6c7 100755 --- a/components/include/external_sdk_server.h +++ b/components/include/external_sdk_server.h @@ -24,7 +24,8 @@ class ExternalSdkServer : public Component, Singleton::Provide, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: ExternalSdkServer(); diff --git a/components/security_apps/local_policy_mgmt_gen/include/ingress_data.h b/components/security_apps/local_policy_mgmt_gen/include/ingress_data.h index 16d7e78..2403290 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/ingress_data.h +++ b/components/security_apps/local_policy_mgmt_gen/include/ingress_data.h @@ -79,6 +79,7 @@ class DefaultBackend { public: void load(cereal::JSONInputArchive &); + bool doesExist() const; private: bool is_exists = false; @@ -90,6 +91,7 @@ public: void load(cereal::JSONInputArchive &archive_in); const std::vector & getRules() const; + bool doesDefaultBackendExist() const; private: std::string ingress_class_name; diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_log_trigger.h b/components/security_apps/local_policy_mgmt_gen/include/new_log_trigger.h index a39194a..d365c58 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_log_trigger.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_log_trigger.h @@ -129,7 +129,7 @@ public: bool shouldBeautifyLogs() const; bool getCloud() const; - bool isK8SNeeded() const; + bool isContainerNeeded() const; bool isCefNeeded() const; bool isSyslogNeeded() const; const std::string & getSyslogServerIpv4Address() const; @@ -140,7 +140,7 @@ private: const NewLoggingService & getCefServiceData() const; bool cloud = false; - bool k8s_service = false; + bool container_service = false; bool agent_local = true; bool beautify_logs = true; NewLoggingService syslog_service; diff --git a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h index 7575720..c760f6e 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h +++ b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h @@ -111,7 +111,7 @@ private: SecurityAppsWrapper security_apps; }; -class PolicyMakerUtils +class PolicyMakerUtils : Singleton::Consume { public: std::string proccesSingleAppsecPolicy( diff --git a/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h b/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h index dc4998d..9e47a68 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/triggers_section.h @@ -39,7 +39,7 @@ public: bool _logToAgent, bool _logToCef, bool _logToCloud, - bool _logToK8sService, + bool _logToContainerService, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -73,7 +73,7 @@ private: bool logToAgent; bool logToCef; bool logToCloud; - bool logToK8sService; + bool logToContainerService; bool logToSyslog; bool responseBody; bool tpDetect; @@ -258,7 +258,7 @@ public: bool shouldBeautifyLogs() const; bool getCloud() const; - bool isK8SNeeded() const; + bool isContainerNeeded() const; bool isCefNeeded() const; bool isSyslogNeeded() const; const std::string & getSyslogServerIpv4Address() const; @@ -269,7 +269,7 @@ private: const LoggingService & getCefServiceData() const; bool cloud = false; - bool k8s_service = false; + bool container_service = false; bool agent_local = true; bool beautify_logs = true; LoggingService syslog_service; diff --git a/components/security_apps/local_policy_mgmt_gen/ingress_data.cc b/components/security_apps/local_policy_mgmt_gen/ingress_data.cc index 8be6f1d..2a6128f 100755 --- a/components/security_apps/local_policy_mgmt_gen/ingress_data.cc +++ b/components/security_apps/local_policy_mgmt_gen/ingress_data.cc @@ -86,6 +86,12 @@ DefaultBackend::load(cereal::JSONInputArchive &) is_exists = true; } +bool +DefaultBackend::doesExist() const +{ + return is_exists; +} + void IngressSpec::load(cereal::JSONInputArchive &archive_in) { @@ -101,6 +107,12 @@ IngressSpec::getRules() const return rules; } +bool +IngressSpec::doesDefaultBackendExist() const +{ + return default_backend.doesExist(); +} + void SingleIngressData::load(cereal::JSONInputArchive &archive_in) { diff --git a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc index 33f4522..57a81a2 100644 --- a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc @@ -532,6 +532,16 @@ K8sPolicyUtils::createPolicy( map &annotations_values, const SingleIngressData &item) const { + if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) { + policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy; + } + if (item.getSpec().doesDefaultBackendExist()) { + dbgTrace(D_LOCAL_POLICY) + << "Inserting Any host rule to the specific asset set"; + K ingress_rule = K("*"); + policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule); + } + for (const IngressDefinedRule &rule : item.getSpec().getRules()) { string url = rule.getHost(); for (const IngressRulePath &uri : rule.getPathsWrapper().getRulePaths()) { @@ -544,14 +554,12 @@ K8sPolicyUtils::createPolicy( << uri.getPath() << "'"; K ingress_rule = K(url + uri.getPath()); - appsec_policy.addSpecificRule(ingress_rule); + policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule); } } } - policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy; } - std::tuple, map> K8sPolicyUtils::createAppsecPoliciesFromIngresses() { diff --git a/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc b/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc index a040f65..dd15280 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc @@ -126,6 +126,7 @@ NewAppsecPolicySpec::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec policy spec"; parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); parseAppsecJSONKey("default", default_rule, archive_in); + default_rule.setHost("*"); parseAppsecJSONKey>("specificRules", specific_rules, archive_in); } diff --git a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc index 8cb488f..28972c2 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc @@ -183,7 +183,9 @@ NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) auto mode = Singleton::Consume::by()->getOrchestrationMode(); auto env_type = Singleton::Consume::by()->getEnvType(); bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); - parseAppsecJSONKey("k8s-service", k8s_service, archive_in, k8s_service_default); + // BC try load previous name. TODO: update CRD + parseAppsecJSONKey("k8s-service", container_service, archive_in, k8s_service_default); + parseAppsecJSONKey("container-service", container_service, archive_in, container_service); NewStdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); @@ -224,9 +226,9 @@ NewAppsecTriggerLogDestination::getCloud() const } bool -NewAppsecTriggerLogDestination::isK8SNeeded() const +NewAppsecTriggerLogDestination::isContainerNeeded() const { - return k8s_service; + return container_service; } bool diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index 2d54e7d..f086905 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -538,7 +538,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp bool webHeaders = trigger_spec.getAppsecTriggerExtendedLogging().isHttpHeaders(); bool webBody = trigger_spec.getAppsecTriggerExtendedLogging().isRequestBody(); bool logToCloud = trigger_spec.getAppsecTriggerLogDestination().getCloud(); - bool logToK8sService = trigger_spec.getAppsecTriggerLogDestination().isK8SNeeded(); + bool logToContainerService = trigger_spec.getAppsecTriggerLogDestination().isContainerNeeded(); bool logToAgent = trigger_spec.getAppsecTriggerLogDestination().isAgentLocal(); bool beautify_logs = trigger_spec.getAppsecTriggerLogDestination().shouldBeautifyLogs(); bool logToCef = trigger_spec.getAppsecTriggerLogDestination().isCefNeeded(); @@ -565,7 +565,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp logToAgent, logToCef, logToCloud, - logToK8sService, + logToContainerService, logToSyslog, responseBody, tpDetect, @@ -1636,7 +1636,9 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy(const string &policy_name, c createPolicyElements(specific_rules, default_rule, appsec_policy, policy_name); // add default rule to policy - createPolicyElementsByRule(default_rule, default_rule, appsec_policy, policy_name); + if (Singleton::Consume::by()->getEnvType() != EnvType::K8S) { + createPolicyElementsByRule(default_rule, default_rule, appsec_policy, policy_name); + } } // LCOV_EXCL_START Reason: no test exist @@ -1659,11 +1661,13 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy( - default_rule, - default_rule, - appsec_policy, - policy_name); + if (Singleton::Consume::by()->getEnvType() != EnvType::K8S) { + createPolicyElementsByRule( + default_rule, + default_rule, + appsec_policy, + policy_name); + } } // LCOV_EXCL_STOP diff --git a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc index e12b323..6659d52 100755 --- a/components/security_apps/local_policy_mgmt_gen/triggers_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/triggers_section.cc @@ -30,7 +30,7 @@ LogTriggerSection::LogTriggerSection( bool _logToAgent, bool _logToCef, bool _logToCloud, - bool _logToK8sService, + bool _logToContainerService, bool _logToSyslog, bool _responseBody, bool _tpDetect, @@ -55,7 +55,7 @@ LogTriggerSection::LogTriggerSection( logToAgent(_logToAgent), logToCef(_logToCef), logToCloud(_logToCloud), - logToK8sService(_logToK8sService), + logToContainerService(_logToContainerService), logToSyslog(_logToSyslog), responseBody(_responseBody), tpDetect(_tpDetect), @@ -101,7 +101,7 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("logToAgent", logToAgent), cereal::make_nvp("logToCef", logToCef), cereal::make_nvp("logToCloud", logToCloud), - cereal::make_nvp("logToK8sService", logToK8sService), + cereal::make_nvp("logToContainerService", logToContainerService), cereal::make_nvp("logToSyslog", logToSyslog), cereal::make_nvp("responseBody", responseBody), cereal::make_nvp("responseCode", false), @@ -396,7 +396,9 @@ AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in) auto mode = Singleton::Consume::by()->getOrchestrationMode(); auto env_type = Singleton::Consume::by()->getEnvType(); bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S); - parseAppsecJSONKey("k8s-service", k8s_service, archive_in, k8s_service_default); + // BC try load previous name. TODO: update CRD + parseAppsecJSONKey("k8s-service", container_service, archive_in, k8s_service_default); + parseAppsecJSONKey("container-service", container_service, archive_in, container_service); StdoutLogging stdout_log; parseAppsecJSONKey("stdout", stdout_log, archive_in); @@ -437,9 +439,9 @@ AppsecTriggerLogDestination::getCloud() const } bool -AppsecTriggerLogDestination::isK8SNeeded() const +AppsecTriggerLogDestination::isContainerNeeded() const { - return k8s_service; + return container_service; } bool diff --git a/components/security_apps/orchestration/CMakeLists.txt b/components/security_apps/orchestration/CMakeLists.txt index 5c067b8..5e0d106 100755 --- a/components/security_apps/orchestration/CMakeLists.txt +++ b/components/security_apps/orchestration/CMakeLists.txt @@ -15,5 +15,6 @@ add_subdirectory(health_check) add_subdirectory(health_check_manager) add_subdirectory(updates_process_reporter) add_subdirectory(env_details) +add_subdirectory(external_sdk_server) #add_subdirectory(orchestration_ut) diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index 34bbd92..8a4d8bf 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -142,7 +142,7 @@ DetailsResolver::Impl::isCloudStorageEnabled() { auto cloud_storage_mode_override = getProfileAgentSetting("agent.cloudStorage.enabled"); if (cloud_storage_mode_override.ok()) { - dbgInfo(D_ORCHESTRATOR) << "Received cloud-storage mode override: " << *cloud_storage_mode_override; + dbgDebug(D_ORCHESTRATOR) << "Received cloud-storage mode override: " << *cloud_storage_mode_override; return *cloud_storage_mode_override; } diff --git a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc index 604cec9..ac6e4b2 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc @@ -142,7 +142,7 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const shared_ptr in_file = Singleton::Consume::by()->fileStreamWrapper(path); if (!in_file->is_open()) { - dbgWarning(D_AGENT_DETAILS) << "Could not open file for processing. Path: " << path; + dbgDebug(D_AGENT_DETAILS) << "Could not open file for processing. Path: " << path; continue; } diff --git a/components/security_apps/orchestration/external_sdk_server/CMakeLists.txt b/components/security_apps/orchestration/external_sdk_server/CMakeLists.txt new file mode 100644 index 0000000..dab2c6f --- /dev/null +++ b/components/security_apps/orchestration/external_sdk_server/CMakeLists.txt @@ -0,0 +1,4 @@ +include_directories(${PROJECT_SOURCE_DIR}/core/external_sdk/) + +add_library(external_sdk_server external_sdk_server.cc) +add_subdirectory(external_sdk_server_ut) diff --git a/components/security_apps/orchestration/external_sdk_server/external_sdk_server.cc b/components/security_apps/orchestration/external_sdk_server/external_sdk_server.cc new file mode 100644 index 0000000..4200443 --- /dev/null +++ b/components/security_apps/orchestration/external_sdk_server/external_sdk_server.cc @@ -0,0 +1,348 @@ +#include "external_sdk_server.h" + +#include "external_agent_sdk.h" +#include "log_generator.h" +#include "rest_server.h" +#include "generic_metric.h" +#include "customized_cereal_map.h" +#include "report/log_rest.h" + +using namespace std; + +USE_DEBUG_FLAG(D_EXTERNAL_SDK_USER); +USE_DEBUG_FLAG(D_EXTERNAL_SDK_SERVER); + +class ExternalSdkRest : public ServerRest +{ +public: + void + doCall() override + { + dbgFlow(D_EXTERNAL_SDK_SERVER); + Maybe sdk_event_type = convertToEnum(event_type.get()); + if (!sdk_event_type.ok()) { + dbgWarning(D_EXTERNAL_SDK_SERVER) << "Received illegal event type. Type : " << event_type.get(); + throw JsonError("Illegal event type provided"); + } + dbgDebug(D_EXTERNAL_SDK_SERVER) + << "Handling a new external sdk api call event. Type : " + << convertApiTypeToString(sdk_event_type.unpack()); + + I_ExternalSdkServer *sdk_server = Singleton::Consume::from(); + switch(sdk_event_type.unpack()) { + case SdkApiType::SendCodeEvent: { + if (!file.isActive()) { + throw JsonError("File was not provided for code event"); + } + if (!func.isActive()) { + throw JsonError("Function was not provided for code event"); + } + if (!line.isActive()) { + throw JsonError("Line path was not provided for code event"); + } + if (!trace_id.isActive()) { + throw JsonError("Trace ID was not provided for code event"); + } + if (!span_id.isActive()) { + throw JsonError("Span ID was not provided for code event"); + } + if (!message.isActive()) { + throw JsonError("Message was not provided for code event"); + } + sdk_server->sendDebug( + file.get(), + func.get(), + line.get(), + getDebugLevel(), + trace_id.get(), + span_id.get(), + message.get(), + additional_fields.isActive() ? additional_fields.get() : map() + ); + return; + } + case SdkApiType::SendEventDrivenEvent: { + if (!event_name.isActive()) { + throw JsonError("Event name was not provided for event"); + } + sdk_server->sendLog( + event_name.get(), + getAudience(), + getSeverity(), + getPriority(), + tag.get(), + additional_fields.isActive() ? additional_fields.get() : map() + ); + return; + } + case SdkApiType::SendGetConfigRequest: { + if (!config_path.isActive()) { + throw JsonError("Config path was not provided for get configuration event"); + } + Maybe config_val = sdk_server->getConfigValue(config_path.get()); + config_value = config_val.ok() ? config_val.unpack() : ""; + return; + } + case SdkApiType::SendPeriodicEvent: { + if (!event_name.isActive()) { + throw JsonError("Event name was not provided for periodic event"); + } + if (!service_name.isActive()) { + throw JsonError("Service name was not provided for periodic event"); + } + sdk_server->sendMetric( + event_name, + service_name, + getAudienceTeam(), + ReportIS::IssuingEngine::AGENT_CORE, + additional_fields.isActive() ? additional_fields.get() : map() + ); + return; + } + default: { + dbgError(D_EXTERNAL_SDK_SERVER) << "Received illegal event type. Type : " << event_type.get(); + } + } + } + +private: + static string + convertApiTypeToString(SdkApiType type) + { + static const EnumArray api_type_string { + "Code Event", + "Periodic Event", + "Event Driven", + "Get Configuration", + }; + return api_type_string[type]; + } + + Debug::DebugLevel + getDebugLevel() + { + static const map debug_levels = { + {0, Debug::DebugLevel::TRACE}, + {1, Debug::DebugLevel::DEBUG}, + {2, Debug::DebugLevel::INFO}, + {3, Debug::DebugLevel::WARNING}, + {4, Debug::DebugLevel::ERROR} + }; + if (!debug_level.isActive()) { + throw JsonError("Debug level was not provided for code event"); + } + auto level = debug_levels.find(debug_level.get()); + if(level == debug_levels.end()) { + throw JsonError("Illegal debug level provided"); + } + + return level->second; + } + + ReportIS::Severity + getSeverity() + { + if (!severity.isActive()) { + throw JsonError("Event severity was not provided for periodic event"); + } + switch (severity.get()) { + case EventSeverity::SeverityCritical: return ReportIS::Severity::CRITICAL; + case EventSeverity::SeverityHigh: return ReportIS::Severity::HIGH; + case EventSeverity::SeverityMedium: return ReportIS::Severity::MEDIUM; + case EventSeverity::SeverityLow: return ReportIS::Severity::LOW; + case EventSeverity::SeverityInfo: return ReportIS::Severity::INFO; + default: + throw JsonError("Illegal event severity provided"); + } + } + + ReportIS::Priority + getPriority() + { + if (!priority.isActive()) { + throw JsonError("Event priority was not provided"); + } + switch (priority.get()) { + case EventPriority::PriorityUrgent: return ReportIS::Priority::URGENT; + case EventPriority::PriorityHigh: return ReportIS::Priority::HIGH; + case EventPriority::PriorityMedium: return ReportIS::Priority::MEDIUM; + case EventPriority::PriorityLow: return ReportIS::Priority::LOW; + default: + throw JsonError("Illegal event priority provided"); + } + } + + ReportIS::Audience + getAudience() + { + if (!audience.isActive()) { + throw JsonError("Event audience was not provided"); + } + switch (audience.get()) { + case EventAudience::AudienceSecurity: return ReportIS::Audience::SECURITY; + case EventAudience::AudienceInternal: return ReportIS::Audience::INTERNAL; + default: + throw JsonError("Illegal event audience provided"); + } + } + + ReportIS::AudienceTeam + getAudienceTeam() + { + if (!team.isActive()) { + throw JsonError("Event audience team was not provided"); + } + switch (team.get()) { + case EventAudienceTeam::AudienceTeamAgentCore: return ReportIS::AudienceTeam::AGENT_CORE; + case EventAudienceTeam::AudienceTeamIot: return ReportIS::AudienceTeam::IOT_NEXT; + case EventAudienceTeam::AudienceTeamWaap: return ReportIS::AudienceTeam::WAAP; + case EventAudienceTeam::AudienceTeamAgentIntelligence: return ReportIS::AudienceTeam::AGENT_INTELLIGENCE; + default: + throw JsonError("Illegal event audience team provided"); + } + } + + using additional_fields_map = map; + C2S_LABEL_PARAM(int, event_type, "eventType"); + C2S_LABEL_OPTIONAL_PARAM(additional_fields_map, additional_fields, "additionalFields"); + C2S_LABEL_OPTIONAL_PARAM(string, event_name, "eventName"); + C2S_LABEL_OPTIONAL_PARAM(string, service_name, "serviceName"); + C2S_OPTIONAL_PARAM(int, team); + C2S_OPTIONAL_PARAM(int, audience); + C2S_OPTIONAL_PARAM(int, severity); + C2S_OPTIONAL_PARAM(int, priority); + C2S_OPTIONAL_PARAM(string, tag); + C2S_OPTIONAL_PARAM(string, file); + C2S_OPTIONAL_PARAM(string, func); + C2S_OPTIONAL_PARAM(int, line); + C2S_LABEL_OPTIONAL_PARAM(int, debug_level, "debugLevel"); + C2S_LABEL_OPTIONAL_PARAM(string, trace_id, "traceId"); + C2S_LABEL_OPTIONAL_PARAM(string, span_id, "spanId"); + C2S_OPTIONAL_PARAM(string, message); + C2S_LABEL_OPTIONAL_PARAM(string, config_path, "configPath"); + S2C_LABEL_OPTIONAL_PARAM(string, config_value, "configValue"); +}; + +class ExternalSdkServer::Impl + : + public Singleton::Provide::From +{ +public: + void + init() + { + auto rest = Singleton::Consume::by(); + rest->addRestCall(RestAction::ADD, "sdk-call"); + } + + void + sendLog( + const string &event_name, + ReportIS::Audience audience, + ReportIS::Severity severity, + ReportIS::Priority priority, + const string &tag_string, + const map &additional_fields) + { + Maybe tag = TagAndEnumManagement::convertStringToTag(tag_string); + set tags; + if (tag.ok()) tags.insert(tag.unpack()); + LogGen log(event_name, audience, severity, priority, tags); + for (const auto &field : additional_fields) { + log << LogField(field.first, field.second); + } + } + + void + sendDebug( + const string &file_name, + const string &function_name, + unsigned int line_number, + Debug::DebugLevel debug_level, + const string &trace_id, + const string &span_id, + const string &message, + const map &additional_fields) + { + (void)trace_id; + (void)span_id; + Debug debug(file_name, function_name, line_number, debug_level, D_EXTERNAL_SDK_USER); + debug.getStreamAggr() << message; + bool is_first_key = true; + for (const auto &field : additional_fields) { + if (is_first_key) { + is_first_key = false; + debug.getStreamAggr() << ". "; + } else { + debug.getStreamAggr() << ", "; + } + debug.getStreamAggr() << "\"" << field.first << "\": \"" << field.second << "\""; + } + } + + void + sendMetric( + const string &event_title, + const string &service_name, + ReportIS::AudienceTeam team, + ReportIS::IssuingEngine issuing_engine, + const map &additional_fields) + { + ScopedContext ctx; + ctx.registerValue("Service Name", service_name); + + set tags; + Report metric_to_fog( + event_title, + Singleton::Consume::by()->getWalltime(), + ReportIS::Type::PERIODIC, + ReportIS::Level::LOG, + ReportIS::LogLevel::INFO, + ReportIS::Audience::INTERNAL, + team, + ReportIS::Severity::INFO, + ReportIS::Priority::LOW, + chrono::seconds(0), + LogField("agentId", Singleton::Consume::by()->getAgentId()), + tags, + ReportIS::Tags::INFORMATIONAL, + issuing_engine + ); + + for (const auto &field : additional_fields) { + metric_to_fog << LogField(field.first, field.second); + } + + LogRest metric_client_rest(metric_to_fog); + + string fog_metric_uri = getConfigurationWithDefault("/api/v1/agents/events", "metric", "fogMetricUri"); + Singleton::Consume::by()->sendAsyncMessage( + HTTPMethod::POST, + fog_metric_uri, + metric_client_rest, + MessageCategory::METRIC, + MessageMetadata(), + false + ); + } + + Maybe + getConfigValue(const string &config_path) + { + auto config_val = getProfileAgentSetting(config_path); + if (!config_val.ok()) { + stringstream error; + error << "Failed to get configuration. Config path: " << config_path << ", Error: " << config_val.getErr(); + return genError(error.str()); + } + return config_val.unpack(); + } +}; + +ExternalSdkServer::ExternalSdkServer() : Component("ExternalSdkServer"), pimpl(make_unique()) {} +ExternalSdkServer::~ExternalSdkServer() {} + +void ExternalSdkServer::init() { pimpl->init(); } +void ExternalSdkServer::fini() {} + +void ExternalSdkServer::preload() {} diff --git a/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/CMakeLists.txt b/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/CMakeLists.txt new file mode 100644 index 0000000..da4d30e --- /dev/null +++ b/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/CMakeLists.txt @@ -0,0 +1,7 @@ +link_directories(${BOOST_ROOT}/lib) + +add_unit_test( + external_sdk_server_ut + "external_sdk_server_ut.cc" + "external_sdk_server;mainloop;singleton;rest;environment;time_proxy;logging;event_is;metric;-lboost_context;agent_details;-lboost_regex;messaging;" +) diff --git a/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/external_sdk_server_ut.cc b/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/external_sdk_server_ut.cc new file mode 100644 index 0000000..ab299f8 --- /dev/null +++ b/components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/external_sdk_server_ut.cc @@ -0,0 +1,349 @@ +#include +#include + +#include "external_sdk_server.h" + +#include "cptest.h" +#include "mock/mock_rest_api.h" +#include "mock/mock_messaging.h" +#include "mock/mock_logging.h" +#include "mock/mock_time_get.h" +#include "config.h" +#include "config_component.h" +#include "agent_details.h" + +using namespace std; +using namespace testing; + +class ExternalSdkServerTest : public Test +{ +public: + ExternalSdkServerTest() + { + EXPECT_CALL(rest_mocker, mockRestCall(RestAction::ADD, "sdk-call", _)).WillOnce( + WithArg<2>( + Invoke( + [this](const unique_ptr &rest_ptr) + { + mock_sdk_rest = rest_ptr->getRest(); + return true; + } + ) + ) + ); + + sdk_server.preload(); + sdk_server.init(); + i_sdk = Singleton::Consume::from(sdk_server); + } + + ~ExternalSdkServerTest() + { + sdk_server.fini(); + } + + ExternalSdkServer sdk_server; + NiceMock mock_timer; + StrictMock messaging_mocker; + StrictMock rest_mocker; + StrictMock log_mocker; + unique_ptr mock_sdk_rest; + I_ExternalSdkServer *i_sdk; + ConfigComponent conf; + AgentDetails agent_details; + ::Environment env; +}; + +TEST_F(ExternalSdkServerTest, initTest) +{ +} + +TEST_F(ExternalSdkServerTest, configCall) +{ + Maybe no_conf = i_sdk->getConfigValue("key1"); + EXPECT_FALSE(no_conf.ok()); + string config_json = + "{\n" + "\"agentSettings\": [\n" + "{\n" + "\"id\": \"id1\",\n" + "\"key\": \"key1\",\n" + "\"value\": \"value1\"\n" + "},\n" + "{\n" + "\"id\": \"id1\",\n" + "\"key\": \"key2\",\n" + "\"value\": \"value2\"\n" + "}\n" + "]\n" + "}\n"; + conf.preload(); + istringstream conf_stream(config_json); + ASSERT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(conf_stream)); + + Maybe conf_found = i_sdk->getConfigValue("key1"); + ASSERT_TRUE(conf_found.ok()); + EXPECT_EQ(conf_found.unpack(), "value1"); + + conf_found = i_sdk->getConfigValue("key2"); + ASSERT_TRUE(conf_found.ok()); + EXPECT_EQ(conf_found.unpack(), "value2"); + + stringstream config_call_body; + config_call_body << "{ \"eventType\": 3, \"configPath\": \"key1\" }"; + + Maybe sdk_conf = mock_sdk_rest->performRestCall(config_call_body); + ASSERT_TRUE(sdk_conf.ok()); + EXPECT_EQ( + sdk_conf.unpack(), + "{\n" + " \"configValue\": \"value1\"\n" + "}" + ); +} + +template +string +toJson(const T &obj) +{ + stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + obj.serialize(ar); + } + return ss.str(); +} + +TEST_F(ExternalSdkServerTest, eventDrivenCall) +{ + string generated_log; + EXPECT_CALL(log_mocker, getCurrentLogId()).Times(2).WillRepeatedly(Return(0)); + EXPECT_CALL(log_mocker, sendLog(_)).Times(2).WillRepeatedly( + WithArg<0>( + Invoke( + [&] (const Report &msg) + { + generated_log = toJson(msg); + } + ) + ) + ); + + i_sdk->sendLog( + "my log", + ReportIS::Audience::INTERNAL, + ReportIS::Severity::LOW, + ReportIS::Priority::HIGH, + "IPS", + {{"key1", "value1"}, {"key2", "value2"}} + ); + static const string expected_log = + "{\n" + " \"eventTime\": \"\",\n" + " \"eventName\": \"my log\",\n" + " \"eventSeverity\": \"Low\",\n" + " \"eventPriority\": \"High\",\n" + " \"eventType\": \"Event Driven\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"\",\n" + " \"eventFrequency\": 0,\n" + " \"eventTags\": [\n" + " \"IPS\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"Unnamed Nano Service\"\n" + " },\n" + " \"eventData\": {\n" + " \"logIndex\": 0,\n" + " \"key1\": \"value1\",\n" + " \"key2\": \"value2\"\n" + " }\n" + "}"; + + EXPECT_EQ(generated_log, expected_log); + + string event_call_body = + "{\n" + " \"eventType\": 2,\n" + " \"eventName\": \"my log\",\n" + " \"audience\": 1,\n" + " \"severity\": 3,\n" + " \"priority\": 1,\n" + " \"tag\": \"IPS\",\n" + " \"team\": 3,\n" + " \"additionalFields\": {\n" + " \"key1\": \"value1\",\n" + " \"key2\": \"value2\"\n" + " }\n" + "}"; + + generated_log = ""; + stringstream event_call_stream; + event_call_stream << event_call_body; + EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok()); + EXPECT_EQ(generated_log, expected_log); +} + +TEST_F(ExternalSdkServerTest, periodicEventCall) +{ + string message_body; + EXPECT_CALL( + messaging_mocker, + sendAsyncMessage( + HTTPMethod::POST, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _, + false + ) + ).Times(2).WillRepeatedly(SaveArg<2>(&message_body)); + + i_sdk->sendMetric( + "my metric", + "matrix", + ReportIS::AudienceTeam::AGENT_INTELLIGENCE, + ReportIS::IssuingEngine::AGENT_CORE, + {{"key", "value"}} + ); + + static const string expected_message = + "{\n" + " \"log\": {\n" + " \"eventTime\": \"\",\n" + " \"eventName\": \"my metric\",\n" + " \"eventSeverity\": \"Info\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Periodic\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"Agent Intelligence\",\n" + " \"eventFrequency\": 0,\n" + " \"eventTags\": [\n" + " \"Informational\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"issuingEngine\": \"Agent Core\",\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"matrix\"\n" + " },\n" + " \"eventData\": {\n" + " \"key\": \"value\"\n" + " }\n" + " }\n" + "}"; + + EXPECT_EQ(message_body, expected_message); + + string event_call_body = + "{\n" + " \"eventType\": 1,\n" + " \"eventName\": \"my metric\",\n" + " \"serviceName\": \"matrix\",\n" + " \"team\": 3,\n" + " \"additionalFields\": {\n" + " \"key\": \"value\"\n" + " }\n" + "}"; + + stringstream event_call_stream; + event_call_stream << event_call_body; + + message_body = ""; + EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok()); + EXPECT_EQ(message_body, expected_message); +} + +USE_DEBUG_FLAG(D_EXTERNAL_SDK_USER); +USE_DEBUG_FLAG(D_EXTERNAL_SDK_SERVER); + +TEST_F(ExternalSdkServerTest, codeEventCall) +{ + ostringstream capture_debug; + Debug::setUnitTestFlag(D_EXTERNAL_SDK_SERVER, Debug::DebugLevel::TRACE); + Debug::setUnitTestFlag(D_EXTERNAL_SDK_USER, Debug::DebugLevel::TRACE); + Debug::setNewDefaultStdout(&capture_debug); + + i_sdk->sendDebug( + "file.cc", + "myFunc2", + 42, + Debug::DebugLevel::TRACE, + "123", + "abc", + "h#l1ow w0r!d", + {{"hi", "universe"}} + ); + + EXPECT_THAT( + capture_debug.str(), + HasSubstr( + "[myFunc2@file.cc:42 | >>>] " + "h#l1ow w0r!d. \"hi\": \"universe\"\n" + ) + ); + + + string debug_event = + "{\n" + " \"eventType\": 0,\n" + " \"file\": \"my file\",\n" + " \"func\": \"function_name\",\n" + " \"line\": 42,\n" + " \"debugLevel\": 0,\n" + " \"traceId\": \"\",\n" + " \"spanId\": \"span2323\",\n" + " \"message\": \"some short debug\",\n" + " \"team\": 1,\n" + " \"additionalFields\": {\n" + " \"name\": \"moshe\",\n" + " \"food\": \"bamba\"\n" + " }\n" + "}"; + + stringstream event_call_stream; + event_call_stream << debug_event; + + EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok()); + + EXPECT_THAT( + capture_debug.str(), + HasSubstr( + "[function_name@my file:42 | >>>] " + "some short debug. \"food\": \"bamba\", \"name\": \"moshe\"\n" + ) + ); + + Debug::setNewDefaultStdout(&cout); +} + +TEST_F(ExternalSdkServerTest, ilegalEventCall) +{ + string event_call_body = + "{\n" + " \"eventType\": 7,\n" + " \"eventName\": \"my metric\",\n" + " \"serviceName\": \"matrix\",\n" + " \"team\": 3,\n" + " \"additionalFields\": {\n" + " \"key\": \"value\"\n" + " }\n" + "}"; + + stringstream event_call_stream; + event_call_stream << event_call_body; + + Maybe failed_respond = mock_sdk_rest->performRestCall(event_call_stream); + EXPECT_FALSE(failed_respond.ok()); + EXPECT_EQ(failed_respond.getErr(), "Illegal event type provided"); +} diff --git a/components/security_apps/orchestration/include/updates_process_report.h b/components/security_apps/orchestration/include/updates_process_report.h index f7d2f0f..a8ee765 100644 --- a/components/security_apps/orchestration/include/updates_process_report.h +++ b/components/security_apps/orchestration/include/updates_process_report.h @@ -50,6 +50,8 @@ public: return report.str(); } + UpdatesFailureReason getReason() const { return reason; } + private: UpdatesProcessResult result; UpdatesConfigType type; diff --git a/components/security_apps/orchestration/include/updates_process_reporter.h b/components/security_apps/orchestration/include/updates_process_reporter.h index 75567e4..7ffad8b 100644 --- a/components/security_apps/orchestration/include/updates_process_reporter.h +++ b/components/security_apps/orchestration/include/updates_process_reporter.h @@ -34,6 +34,7 @@ private: void sendReoprt(); static std::vector reports; + uint report_failure_count = 0; }; #endif // __UPDATES_PROCESS_REPORTER_H__ diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 4fccec2..16da91c 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1499,7 +1499,7 @@ private: << " minutes from now."; upgrade_delay_time += chrono::minutes(upgrade_delay_interval); } catch (const exception& err) { - dbgInfo(D_ORCHESTRATOR) << "Failed to parse upgrade delay interval."; + dbgWarning(D_ORCHESTRATOR) << "Failed to parse upgrade delay interval."; } } diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 2b21a58..7a25d41 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -413,7 +413,7 @@ ServiceController::Impl::getUpdatedReconfStatus() } if (!maybe_service.unpack().isServiceActive()) { - dbgInfo(D_SERVICE_CONTROLLER) + dbgDebug(D_SERVICE_CONTROLLER) << "Service is not active, removing from registered services list. Service: " << services_reconf_names[service_and_reconf_status.first] << "ID: " @@ -508,7 +508,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() ar(cereal::make_nvp("Registered Services", pending_services)); pending_services.erase("cp-nano-orchestration"); - dbgInfo(D_SERVICE_CONTROLLER) + dbgDebug(D_SERVICE_CONTROLLER) << "Orchestration pending services loaded from file." << " File: " << registered_services_file @@ -516,7 +516,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile() for (const auto &id_service_pair : pending_services) { const auto &service = id_service_pair.second; - dbgInfo(D_SERVICE_CONTROLLER) + dbgDebug(D_SERVICE_CONTROLLER) << "Service name: " << service.getServiceName() << ", Service ID: " @@ -548,14 +548,14 @@ ServiceController::Impl::writeRegisteredServicesToFile() cereal::JSONOutputArchive ar(ss); ar(cereal::make_nvp("Registered Services", registered_services_with_orch)); - dbgInfo(D_SERVICE_CONTROLLER) + dbgDebug(D_SERVICE_CONTROLLER) << "Orchestration registered services file has been updated. File: " << registered_services_file << ". Registered Services:"; for (const auto &id_service_pair : registered_services_with_orch) { const auto &service = id_service_pair.second; - dbgInfo(D_SERVICE_CONTROLLER) + dbgDebug(D_SERVICE_CONTROLLER) << "Service name: " << service.getServiceName() << ", Service ID: " diff --git a/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc b/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc index c9269a0..6a61049 100644 --- a/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc +++ b/components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc @@ -31,9 +31,15 @@ UpdatesProcessReporter::upon(const UpdatesProcessEvent &event) if (event.getReason() == UpdatesFailureReason::CHECK_UPDATE) { if (event.getResult() == UpdatesProcessResult::SUCCESS && reports.empty()) { dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Update proccess finished successfully"; + report_failure_count = 0; return; } dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Update proccess finished with errors"; + report_failure_count++; + if (report_failure_count <= 1) { + reports.clear(); + return; + } reports.emplace_back( UpdatesProcessReport( event.getResult(), @@ -54,18 +60,27 @@ UpdatesProcessReporter::upon(const UpdatesProcessEvent &event) void UpdatesProcessReporter::sendReoprt() { - stringstream all_reports; - all_reports << "Updates process reports:" << endl; + stringstream full_reports; + UpdatesFailureReason failure_reason = UpdatesFailureReason::NONE; + full_reports << "Updates process reports:" << endl; + full_reports << "report failure count:" << report_failure_count << endl; for (const auto &report : reports) { - all_reports << report.toString() << endl; + if (report.getReason() != UpdatesFailureReason::CHECK_UPDATE) { + failure_reason = report.getReason(); + } + full_reports << report.toString() << endl; } reports.clear(); - dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Sending updates process report: " << endl << all_reports.str(); - LogGen( + dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Sending updates process report: " << endl << full_reports.str(); + LogGen log ( "Updates process report", ReportIS::Audience::INTERNAL, ReportIS::Severity::HIGH, ReportIS::Priority::HIGH, ReportIS::Tags::ORCHESTRATOR - ) << LogField("eventMessage", all_reports.str()); + ); + log << LogField("eventMessage", full_reports.str()); + if (failure_reason != UpdatesFailureReason::NONE) { + log.addToOrigin(LogField("eventCategory", convertUpdatesFailureReasonToStr(failure_reason))); + } } diff --git a/components/utils/generic_rulebase/triggers_config.cc b/components/utils/generic_rulebase/triggers_config.cc index 01429ec..2ad465f 100644 --- a/components/utils/generic_rulebase/triggers_config.cc +++ b/components/utils/generic_rulebase/triggers_config.cc @@ -173,7 +173,7 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields); setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams); setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams); - setTriggersFlag("logToK8sService", archive_in, ReportIS::StreamType::JSON_K8S_SVC, active_streams); + setTriggersFlag("logToContainerService", archive_in, ReportIS::StreamType::JSON_CONTAINER_SVC, active_streams); setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams); setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams); setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect); @@ -221,8 +221,8 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) case ReportIS::StreamType::JSON_LOG_FILE: setLogConfiguration(ReportIS::StreamType::JSON_LOG_FILE); break; - case ReportIS::StreamType::JSON_K8S_SVC: - setLogConfiguration(ReportIS::StreamType::JSON_K8S_SVC); + case ReportIS::StreamType::JSON_CONTAINER_SVC: + setLogConfiguration(ReportIS::StreamType::JSON_CONTAINER_SVC); break; case ReportIS::StreamType::SYSLOG: setLogConfiguration(ReportIS::StreamType::SYSLOG, getUrlForSyslog(), syslog_protocol); diff --git a/core/external_sdk/external_agent_sdk.h b/core/external_sdk/external_agent_sdk.h new file mode 100644 index 0000000..f332354 --- /dev/null +++ b/core/external_sdk/external_agent_sdk.h @@ -0,0 +1,87 @@ +#ifndef __EXTERNAL_AGENT_SDK_H__ +#define __EXTERNAL_AGENT_SDK_H__ + +#ifdef __cplusplus +enum class SdkApiType +#else +enum SdkApiType +#endif +{ + SendCodeEvent, + SendPeriodicEvent, + SendEventDrivenEvent, + SendGetConfigRequest, + +#ifndef __cplusplus +}; +#else //__cplusplus + COUNT +}; + +extern "C" +{ +#endif // __cplusplus + +enum DebugLevel { DebugTrace, DebugDebug, DebugInfo, DebugWarning, DebugError }; +enum EventAudience { AudienceSecurity, AudienceInternal }; +enum EventAudienceTeam { AudienceTeamAgentCore, AudienceTeamIot, AudienceTeamWaap, AudienceTeamAgentIntelligence }; +enum EventSeverity { SeverityCritical, SeverityHigh, SeverityMedium, SeverityLow, SeverityInfo }; +enum EventPriority { PriorityUrgent, PriorityHigh, PriorityMedium, PriorityLow }; + +enum SdkReturn { + SdkSuccess = 0, + SdkUninitialized = -1, + IlegalNumOfAdditionData = -2, + EmptyConfigRespond = -3, + InitCurlFailed = -4, + ExecuteCurlFailed = -5, + Non200Respond = -6, + AllocationFailure = -7 +}; + +void initAgentSdk(); +void finiAgentSdk(); + +// Get configuration using path. Output is allocated internally and requires caller to free +enum SdkReturn getAgentConfiguration(const char *configuration_path, char **config_value_output); + +enum SdkReturn +sendPeriodicData( + const char *event_title, + const char *service_name, + enum EventAudienceTeam team, + const char **periodic_data, + int periodic_data_size +); + +enum SdkReturn +sendEventDrivenLog( + const char *event_name, + enum EventAudience audience, + enum EventSeverity severity, + enum EventPriority priority, + const char *tag, + enum EventAudienceTeam team, + const char **event_data, + int event_data_size +); + +enum SdkReturn +sendDebugMessage( + const char *file_name, + const char *function_name, + unsigned int line_number, + enum DebugLevel debug_level, + const char *trace_id, + const char *span_id, + const char *message, + enum EventAudienceTeam team, + const char **event_data, + int event_data_size +); + +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // __EXTERNAL_AGENT_SDK_H__ diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h index 7318b12..5e0407c 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -5,6 +5,7 @@ #include #include "flags.h" +#include "config.h" #include "singleton.h" #include "i_agent_details.h" @@ -118,6 +119,29 @@ public: return headers; } + std::string + getCaPath() const + { + if (!ca_path.empty()) return ca_path; + return getConfigurationWithDefault( + getFilesystemPathConfig() + "/certs/fog.pem", + "message", + "Certificate chain file path" + ); + } + + const std::string & + getClientCertPath() const + { + return client_cert_path; + } + + const std::string & + getClientKeyPath() const + { + return client_key_path; + } + void insertHeader(const std::string &header_key, const std::string &header_val) { @@ -137,6 +161,24 @@ public: is_proxy_set = true; } + void + setCAPath (const std::string &_ca_path) + { + ca_path = _ca_path; + } + + void + setDualAuthenticationSettings + ( + const std::string &_client_cert_path, + const std::string &_client_key_path + ) + { + client_cert_path = _client_cert_path; + client_key_path = _client_key_path; + is_dual_auth = true; + } + void setExternalCertificate(const std::string &_external_certificate) { @@ -161,6 +203,12 @@ public: return is_proxy_set; } + bool + isDualAuth() const + { + return is_dual_auth; + } + bool isToFog() const { @@ -175,18 +223,26 @@ public: cereal::make_nvp("host_name", host_name), cereal::make_nvp("port_num", port_num), cereal::make_nvp("is_proxy_set", is_proxy_set), + cereal::make_nvp("is_dual_auth", is_dual_auth), cereal::make_nvp("headers", headers), cereal::make_nvp("conn_flags", conn_flags), cereal::make_nvp("external_certificate", external_certificate), cereal::make_nvp("should_buffer", should_buffer), - cereal::make_nvp("is_to_fog", is_to_fog) + cereal::make_nvp("is_to_fog", is_to_fog), + cereal::make_nvp("ca_path", ca_path), + cereal::make_nvp("client_cert_path", client_cert_path), + cereal::make_nvp("client_key_path", client_key_path) ); } private: std::string host_name = ""; + std::string ca_path = ""; + std::string client_cert_path = ""; + std::string client_key_path = ""; uint16_t port_num = 0; bool is_proxy_set = false; + bool is_dual_auth = false; std::map headers; Flags conn_flags; MessageProxySettings proxy_settings; diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 6309d89..6358673 100644 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -23,7 +23,7 @@ enum class StreamType { JSON_DEBUG, JSON_FOG, JSON_LOG_FILE, - JSON_K8S_SVC, + JSON_CONTAINER_SVC, SYSLOG, CEF, diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 70fdcf7..fad66d5 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -383,6 +383,11 @@ private: } if (getProfileAgentSettingWithDefault(false, "agent.config.supportInvalidation")) return true; + + if (getSetting("intelligence", "local intelligence server ip").ok()) { + return getProfileAgentSettingWithDefault(true, "agent.config.useLocalIntelligence"); + } + dbgTrace(D_INTELLIGENCE) << "Local intelligence not supported"; return false; diff --git a/core/logging/k8s_svc_stream.cc b/core/logging/k8s_svc_stream.cc index f9bf877..c3286f7 100644 --- a/core/logging/k8s_svc_stream.cc +++ b/core/logging/k8s_svc_stream.cc @@ -22,21 +22,21 @@ const static string default_log_uri = "/api/v1/agents/events"; USE_DEBUG_FLAG(D_REPORT); -K8sSvcStream::K8sSvcStream() +ContainerSvcStream::ContainerSvcStream() : i_msg(Singleton::Consume::by()) { } -K8sSvcStream::~K8sSvcStream() +ContainerSvcStream::~ContainerSvcStream() { } void -K8sSvcStream::sendLog(const Report &log) +ContainerSvcStream::sendLog(const Report &log) { - auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host"); - auto K8sSvc_log_uri = getConfigurationWithDefault(default_log_uri, "Logging", "K8sSvc Log URI"); + auto svc_host = getConfigurationWithDefault(default_host, "Logging", "Container Log host"); + auto svc_log_uri = getConfigurationWithDefault(default_log_uri, "Logging", "Container Log URI"); LogRest rest(log); MessageMetadata rest_req_md(svc_host, 80); @@ -45,7 +45,7 @@ K8sSvcStream::sendLog(const Report &log) bool ok = i_msg->sendSyncMessageWithoutResponse( HTTPMethod::POST, - K8sSvc_log_uri, + svc_log_uri, rest, MessageCategory::LOG, rest_req_md @@ -57,7 +57,7 @@ K8sSvcStream::sendLog(const Report &log) } void -K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only) +ContainerSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only) { dbgFlow(D_REPORT) << "send bulk logs"; @@ -66,15 +66,15 @@ K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only) return; } - auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host"); - auto K8sSvc_log_uri = getConfigurationWithDefault(default_bulk_uri, "Logging", "K8sSvc Bulk Log URI"); + auto svc_host = getConfigurationWithDefault(default_host, "Logging", "Container Log host"); + auto svc_log_uri = getConfigurationWithDefault(default_bulk_uri, "Logging", "Container Bulk Log URI"); MessageMetadata rest_req_md(svc_host, 80); rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); bool ok = i_msg->sendSyncMessageWithoutResponse( HTTPMethod::POST, - K8sSvc_log_uri, + svc_log_uri, logs, MessageCategory::LOG, rest_req_md diff --git a/core/logging/log_streams.h b/core/logging/log_streams.h index 0afa819..b58303a 100644 --- a/core/logging/log_streams.h +++ b/core/logging/log_streams.h @@ -80,11 +80,11 @@ private: I_Messaging *i_msg = nullptr; }; -class K8sSvcStream : public Stream +class ContainerSvcStream : public Stream { public: - K8sSvcStream(); - ~K8sSvcStream(); + ContainerSvcStream(); + ~ContainerSvcStream(); void sendLog(const Report &log) override; void sendLog(const LogBulkRest &logs, bool persistance_only) override; diff --git a/core/logging/logging.cc b/core/logging/logging.cc index f43a208..4382690 100644 --- a/core/logging/logging.cc +++ b/core/logging/logging.cc @@ -270,7 +270,7 @@ private: case StreamType::JSON_DEBUG: return make_shared(); case StreamType::JSON_FOG: return make_shared(); case StreamType::JSON_LOG_FILE: return make_shared(); - case StreamType::JSON_K8S_SVC: return make_shared(); + case StreamType::JSON_CONTAINER_SVC: return make_shared(); case StreamType::SYSLOG: return nullptr; case StreamType::CEF: return nullptr; case StreamType::NONE: return nullptr; @@ -319,7 +319,9 @@ LoggingComp::preload() registerExpectedConfiguration("Logging", "Log file name"); registerExpectedConfiguration("Logging", "Log file line separator"); registerExpectedConfiguration("Logging", "Fog Log URI"); - registerExpectedConfiguration("Logging", "K8sSvc Log host"); + registerExpectedConfiguration("Logging", "Container Log host"); + registerExpectedConfiguration("Logging", "Container Log URI"); + registerExpectedConfiguration("Logging", "Container Bulk Log URI"); registerExpectedConfiguration("Logging", "Syslog IP"); registerExpectedConfiguration("Logging", "Syslog port"); registerExpectedConfiguration("Logging", "CEF IP"); diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc index 3518f09..75b75cf 100644 --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -68,7 +68,7 @@ public: return; } if (should_load_k8s_stream) { - Singleton::Consume::by()->addStream(ReportIS::StreamType::JSON_K8S_SVC); + Singleton::Consume::by()->addStream(ReportIS::StreamType::JSON_CONTAINER_SVC); return; } Singleton::Consume::by()->addStream(ReportIS::StreamType::JSON_DEBUG); diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index ac0e5e5..0e02dee 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -90,6 +90,12 @@ public: if (metadata_flags.isSet(MessageConnectionConfig::IGNORE_SSL_VALIDATION)) { flags.setFlag(ConnectionFlags::IGNORE_SSL_VALIDATION); } + ca_path = metadata.getCaPath(); + if (metadata.isDualAuth()) { + client_cert_path = metadata.getClientCertPath(); + client_key_path = metadata.getClientKeyPath(); + is_dual_auth = true; + } } void @@ -263,20 +269,33 @@ private: SSL_CTX_set_verify(ssl_ctx.get(), SSL_VERIFY_PEER, nullptr); - auto defualt_cert_path = getFilesystemPathConfig() + "/certs/fog.pem"; - auto cert_path = getConfigurationWithDefault(defualt_cert_path, "message", "Certificate chain file path"); - const char *cert = cert_path.c_str(); + if (is_dual_auth) { + dbgTrace(D_CONNECTION) + << "Setting dual authentication." + << "Client cert path: " << client_cert_path + << ", client key path: " << client_key_path; + if (SSL_CTX_use_certificate_file(ssl_ctx.get(), client_cert_path.c_str(), SSL_FILETYPE_PEM) <= 0) { + string error = ERR_error_string(ERR_get_error(), nullptr); + return genError("Error in setting client cert: " + error); + } + if (SSL_CTX_use_PrivateKey_file(ssl_ctx.get(), client_key_path.c_str(), SSL_FILETYPE_PEM) <= 0) { + string error = ERR_error_string(ERR_get_error(), nullptr); + return genError("Error in setting client key: " + error); + } + } + + dbgTrace(D_CONNECTION) << "Setting CA authentication"; auto details_ssl_dir = Singleton::Consume::by()->getOpenSSLDir(); auto openssl_dir = details_ssl_dir.ok() ? *details_ssl_dir : "/usr/lib/ssl/certs/"; auto configured_ssl_dir = getConfigurationWithDefault(openssl_dir, "message", "Trusted CA directory"); const char *ca_dir = configured_ssl_dir.empty() ? nullptr : configured_ssl_dir.c_str(); - if (SSL_CTX_load_verify_locations(ssl_ctx.get(), cert, ca_dir) != 1) { + if (SSL_CTX_load_verify_locations(ssl_ctx.get(), ca_path.c_str(), ca_dir) != 1) { return genError("Failed to load certificate locations"); } - dbgDebug(D_CONNECTION) << "SSL context set successfully. Certificate: " << cert << ", CA dir: " << ca_dir; + dbgDebug(D_CONNECTION) << "SSL context set successfully. Certificate: " << ca_path << ", CA dir: " << ca_dir; return Maybe(); } @@ -457,7 +476,6 @@ private: return BioConnectionStatus::SHOULD_NOT_RETRY; } - Maybe connectToHost() { @@ -654,6 +672,10 @@ private: Flags flags; MessageProxySettings settings; + string ca_path = ""; + string client_cert_path = ""; + string client_key_path = ""; + string connect_message; string certificate; @@ -666,6 +688,7 @@ private: bool lock = false; bool should_close_connection = false; + bool is_dual_auth = false; }; Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata) diff --git a/core/messaging/connection/connection_ut/connection_comp_ut.cc b/core/messaging/connection/connection_ut/connection_comp_ut.cc index bc1bc4f..4fe682b 100644 --- a/core/messaging/connection/connection_ut/connection_comp_ut.cc +++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc @@ -89,6 +89,8 @@ TEST_F(TestConnectionComp, testSetAndGetConnection) Flags conn_flags; conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + conn_metadata.setCAPath("ca.pem"); + conn_metadata.setDualAuthenticationSettings("ca_client.pem", "private_client.key"); auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); ASSERT_TRUE(maybe_connection.ok()); diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc index ed2cc94..a90a4d3 100644 --- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -253,9 +253,13 @@ operator==(const MessageMetadata &one, const MessageMetadata &two) one.getConnectionFlags() == two.getConnectionFlags() && one.getProxySettings() == two.getProxySettings() && one.getExternalCertificate() == two.getExternalCertificate() && + one.getCaPath() == two.getCaPath() && + one.getClientCertPath() == two.getClientCertPath() && + one.getClientKeyPath() == two.getClientKeyPath() && one.getHeaders() == two.getHeaders() && one.shouldBufferMessage() == two.shouldBufferMessage() && - one.isProxySet() == two.isProxySet(); + one.isProxySet() == two.isProxySet() && + one.isDualAuth() == two.isDualAuth(); } TEST_F(TestMessagingComp, testSetFogConnection) diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 824d5e3..173301e 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -150,7 +150,7 @@ TagAndEnumManagement::convertToString(const StreamType &stream_type) case StreamType::JSON_DEBUG: return "JSON Debug stream"; case StreamType::JSON_FOG: return "JSON FOG stream"; case StreamType::JSON_LOG_FILE: return "JSON File stream"; - case StreamType::JSON_K8S_SVC: return "JSON K8S service stream"; + case StreamType::JSON_CONTAINER_SVC: return "JSON K8S service stream"; case StreamType::SYSLOG: return "Syslog stream"; case StreamType::CEF: return "CEF stream"; diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 2b1bdd7..2c2c484 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -30,7 +30,7 @@ target_link_libraries( env_details local_policy_mgmt_gen curl - + external_sdk_server -Wl,--end-group ) diff --git a/nodes/orchestration/main.cc b/nodes/orchestration/main.cc index 8105c51..8e99890 100755 --- a/nodes/orchestration/main.cc +++ b/nodes/orchestration/main.cc @@ -51,6 +51,7 @@ #include "generic_metric.h" #include "tenant_manager.h" #include "local_policy_mgmt_gen.h" +#include "external_sdk_server.h" using namespace std; @@ -72,7 +73,8 @@ main(int argc, char **argv) OrchestrationTools, HealthChecker, HealthCheckManager, - LocalPolicyMgmtGenerator + LocalPolicyMgmtGenerator, + ExternalSdkServer > comps; comps.registerGlobalValue("Nano service API Port Primary", 7777); diff --git a/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s b/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s index 7a873b8..5e3e24a 100755 --- a/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s +++ b/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s @@ -13,6 +13,10 @@ profile_id= cluster_id= latest_policy_version=1 +if [ -f $POLICY_CRDS_PATH ]; then + chmod 644 $POLICY_CRDS_PATH +fi + load_agent_details() { tenant_id=$(awk -F\" '/Tenant ID/{print $4}' /etc/cp/conf/agent_details.json) @@ -29,7 +33,7 @@ get_latest_policy_version() bucket_list=$(curl -s -w "%{http_code}\n" --request GET \ -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Authorization: Bearer ${ra_token}" \ "$var_fog/agents-core/storage/?list-type=2&prefix=${tenant_id}/${profile_id}") - paths_list=$(echo $bucket_list | /etc/cp/bin/yq -p xml | grep "/policy") + paths_list=$(echo $bucket_list | awk -F'|' '/policy-/ {for (i = 1; i <= NF; i++) if ($i ~ /policy/) print $i}') prefix="${tenant_id}/${profile_id}" paths=$(echo $paths_list | tr " " "\n" | grep / ) @@ -257,6 +261,7 @@ usage() echo "Options:" echo " --fog : Namespace with the relevant Helm Chart" echo " --upload_policy_only : Upload policy to the fog, withput changing agent mode" + echo " --debug : Keep the debuging files" exit 255 } @@ -277,6 +282,8 @@ validate_arg_value_exists() fi } +debug_mode="false" + while true; do if [ "$1" = "--token" ]; then validate_arg_value_exists "$1" "$#" @@ -290,6 +297,8 @@ while true; do validate_arg_value_exists "$1" "$#" shift ra_token="$1" + elif [ "$1" = "--debug" ]; then + debug_mode="true" elif [ -z "$1" ]; then break fi @@ -305,5 +314,8 @@ upload_crds_to_the_cloud if [ "$?" = "0" ]; then echo "SUCCESS" fi +if [ "$debug_mode" = "false" ]; then + rm $POLICY_CRDS_PATH +fi exit 0