github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-30 03:34:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jan_31_2024-Dev

Browse Source
This commit is contained in:
Ned Wright
2024-01-31 17:34:53 +00:00
parent 752a5785f0
commit 6d67818a94
376 changed files with 8101 additions and 7064 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
components/security_apps/orchestration/downloader/curl_client.cc
View File

@@ -55,7 +55,7 @@ HttpCurl::HttpCurl(
const string &_bearer,
const Maybe<string> &proxy_url,
const Maybe<uint16_t> &proxy_port,
const Maybe<string> &proxy_auth)
const Maybe<string> &_proxy_auth)
:
url(_url),
out_file(_out_file),
@@ -85,10 +85,10 @@ HttpCurl::HttpCurl(
proxy = proxy_url.unpack() + ":" + to_string(proxy_port.unpack());
}
}
if (proxy_auth.ok())
if (_proxy_auth.ok())
{
I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<HttpCurl>();
proxy_credentials = "Proxy-Authorization: Basic " + encryptor->base64Encode(proxy_auth.unpack());
proxy_auth = "Proxy-Authorization: Basic " + encryptor->base64Encode(_proxy_auth.unpack());
}
}
@@ -98,7 +98,7 @@ HttpCurl::HttpCurl(const HttpCurl &other)
out_file(other.out_file),
bearer(other.bearer),
proxy(other.proxy),
proxy_credentials(other.proxy_credentials),
proxy_auth(other.proxy_auth),
curl(unique_ptr<CURL, function<void(CURL *)>>(curl_easy_init(), curl_easy_cleanup))
{
}
@@ -133,9 +133,9 @@ HttpCurl::setCurlOpts(long timeout, HTTP_VERSION http_version)
if (!proxy.empty())
{
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.c_str());
if (!proxy_credentials.empty())
if (!proxy_auth.empty())
{
proxy_headers = curl_slist_append(proxy_headers, proxy_credentials.c_str());
proxy_headers = curl_slist_append(proxy_headers, proxy_auth.c_str());
//Apply proxy headers
curl_easy_setopt(curl_handle, CURLOPT_PROXYHEADER, proxy_headers);
}
@@ -330,9 +330,9 @@ HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version)
if (!proxy.empty())
{
curl_easy_setopt(curl_handle, CURLOPT_PROXY, proxy.c_str());
if (!proxy_credentials.empty())
if (!proxy_auth.empty())
{
proxy_headers = curl_slist_append(proxy_headers, proxy_credentials.c_str());
proxy_headers = curl_slist_append(proxy_headers, proxy_auth.c_str());
//Apply proxy headers
curl_easy_setopt(curl_handle, CURLOPT_PROXYHEADER, proxy_headers);
}

2
components/security_apps/orchestration/downloader/curl_client.h
View File

@@ -83,7 +83,7 @@ protected:
std::ofstream &out_file;
std::string bearer;
std::string proxy;
std::string proxy_credentials;
std::string proxy_auth;
std::unique_ptr<CURL, std::function<void(CURL *)>> curl;
std::string curl_url;
};

39
components/security_apps/orchestration/downloader/downloader.cc
View File

@@ -81,6 +81,8 @@ public:
const string &service_name
) const override;
Maybe<string> checkIfFileExists(const Package &package) const override;
void removeDownloadFile(const string &file_name) const override;
void createTenantProfileMap();
string getProfileFromMap(const string &tenant_id) const override;
@@ -194,12 +196,18 @@ Downloader::Impl::downloadVirtualFileFromFog(
static const string error_text = "error";
map<pair<string, string>, string> res;
string general_file_path = dir_path + "/" + resourse_file.getFileName() + "_general.download";
I_UpdateCommunication *update_communication = Singleton::Consume<I_UpdateCommunication>::by<Downloader>();
auto downloaded_data = update_communication->downloadAttributeFile(resourse_file);
auto downloaded_data = update_communication->downloadAttributeFile(resourse_file, general_file_path);
if (!downloaded_data.ok()) return downloaded_data.passErr();
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<Downloader>();
Maybe<string> file_content = orchestration_tools->readFile(general_file_path);
if (!file_content.ok()) return file_content.passErr();
Document document;
document.Parse(downloaded_data.unpack().c_str());
document.Parse(file_content.unpack().c_str());
if (document.HasParseError()) {
dbgWarning(D_ORCHESTRATOR) << "JSON file is not valid";
return genError("JSON file is not valid.");
@@ -241,7 +249,6 @@ Downloader::Impl::downloadVirtualFileFromFog(
rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
artifact_data->value.Accept(writer);
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<Downloader>();
if (orchestration_tools->writeFile(buffer.GetString(), file_path)) {
res.insert({{tenant_id, profile_id}, file_path});
}
@@ -324,6 +331,24 @@ Downloader::Impl::downloadFileFromURL(
return file_path;
}
Maybe<string>
Downloader::Impl::checkIfFileExists(const Package &package) const
{
string file_name = package.getName() + ".download";
Maybe<string> maybe_path = dir_path + "/" + file_name;
return validateChecksum(package.getChecksum(), package.getChecksumType(), maybe_path);
}
void
Downloader::Impl::removeDownloadFile(const string &file_name) const
{
string file_path = dir_path + "/" + file_name + ".download";
dbgInfo(D_ORCHESTRATOR) << "Removing download file " << file_path;
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<Downloader>();
orchestration_tools->removeFile(file_path);
}
Maybe<string>
Downloader::Impl::validateChecksum(
const string &checksum,
@@ -355,13 +380,11 @@ Downloader::Impl::downloadFileFromFogByHTTP(const GetResourceFile &resourse_file
dbgInfo(D_ORCHESTRATOR) << "Downloading file from fog. File: " << resourse_file.getFileName();
I_UpdateCommunication *update_communication = Singleton::Consume<I_UpdateCommunication>::by<Downloader>();
auto downloaded_file = update_communication->downloadAttributeFile(resourse_file);
auto downloaded_file = update_communication->downloadAttributeFile(resourse_file, file_path);
if (!downloaded_file.ok()) return genError(downloaded_file.getErr());
dbgInfo(D_ORCHESTRATOR) << "Download completed. File: " << resourse_file.getFileName();
I_OrchestrationTools *orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<Downloader>();
if (orchestration_tools->writeFile(downloaded_file.unpack(), file_path)) return file_path;
return genError("Failed to write the attribute file. File: " + file_name);
dbgInfo(D_ORCHESTRATOR) << "Download completed. File: " << resourse_file.getFileName();
return file_path;
}
Maybe<string>

81
components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc
View File

@@ -18,6 +18,7 @@ class DownloaderTest : public Test
public:
DownloaderTest()
{
Debug::setUnitTestFlag(D_ORCHESTRATOR, Debug::DebugLevel::TRACE);
setConfiguration<string>("/tmp", "orchestration", "Default file download path");
EXPECT_CALL(mock_orchestration_tools, createDirectory("/tmp")).WillOnce(Return(true));
downloader.init();
@@ -44,15 +45,14 @@ TEST_F(DownloaderTest, downloadFileFromFog)
GetResourceFile resourse_file(GetResourceFile::ResourceFileType::VIRTUAL_SETTINGS);
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file, "/tmp/virtualSettings.download"))
.WillOnce(Return(fog_response));
EXPECT_CALL(
mock_orchestration_tools,
calculateChecksum(Package::ChecksumTypes::SHA256, "/tmp/virtualSettings.download")
).WillOnce(Return(string("123")));
EXPECT_CALL(mock_orchestration_tools, writeFile(fog_response, "/tmp/virtualSettings.download", false))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/virtualSettings.download")).WillOnce(Return(true));
Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
@@ -71,7 +71,10 @@ TEST_F(DownloaderTest, downloadFileFromFogFailure)
Maybe<string> fog_response(genError("Failed to download"));
GetResourceFile resourse_file(GetResourceFile::ResourceFileType::SETTINGS);
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(
mock_communication,
downloadAttributeFile(resourse_file, "/tmp/settings.download")
).WillOnce(Return(fog_response));
Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
checksum,
@@ -124,6 +127,53 @@ TEST_F(DownloaderTest, registerConfig)
env.fini();
}
TEST_F(DownloaderTest, checkIfFileExists)
{
string local_file_path = "/tmp/test_file.sh";
string url = "file://" + local_file_path;
string dir_path = getConfigurationWithDefault<string>(
"/tmp/orchestration_downloads",
"orchestration",
"Default file download path"
);
string manifest =
"{"
" \"packages\": ["
" {"
" \"name\": \"test\","
" \"version\": \"c\","
" \"download-path\": \"http://172.23.92.135/my.sh\","
" \"relative-path\": \"\","
" \"checksum-type\": \"sha1sum\","
" \"checksum\": \"1234\","
" \"package-type\": \"service\","
" \"require\": []"
" }"
" ]"
"}";
vector<Package> manifest_services;
std::stringstream os(manifest);
cereal::JSONInputArchive archive_in(os);
archive_in(manifest_services);
string service_name = "test";
string file_name = service_name + ".download";
string file_path = dir_path + "/" + file_name;
string checksum = "1234";
Package::ChecksumTypes checksum_type = Package::ChecksumTypes::SHA1;
EXPECT_CALL(mock_orchestration_tools, calculateChecksum(checksum_type, file_path)).WillOnce(Return(checksum));
i_downloader->checkIfFileExists(manifest_services[0]);
}
TEST_F(DownloaderTest, removeDownloadFile)
{
string file_path = "/tmp/package.download";
EXPECT_CALL(mock_orchestration_tools, removeFile(file_path)).WillOnce(Return(true));
i_downloader->removeDownloadFile("package");
}
TEST_F(DownloaderTest, downloadWithBadChecksum)
{
string local_file_path = "/tmp/test_file.sh";
@@ -181,10 +231,9 @@ TEST_F(DownloaderTest, downloadEmptyFileFromFog)
GetResourceFile resourse_file(GetResourceFile::ResourceFileType::MANIFEST);
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file, "/tmp/manifest.download"))
.WillOnce(Return(fog_response));
EXPECT_CALL(mock_orchestration_tools, writeFile(fog_response, "/tmp/manifest.download", false))
.WillOnce(Return(true));
EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/manifest.download")).WillOnce(Return(false));
EXPECT_CALL(
@@ -340,7 +389,13 @@ TEST_F(DownloaderTest, download_virtual_policy)
" ]\n"
"}";
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(
mock_communication,
downloadAttributeFile(resourse_file, "/tmp/virtualPolicy_general.download"))
.WillOnce(Return(fog_response));
EXPECT_CALL(mock_orchestration_tools, readFile("/tmp/virtualPolicy_general.download"))
.WillOnce(Return(fog_response));
EXPECT_CALL(
mock_orchestration_tools,
@@ -428,7 +483,15 @@ TEST_F(DownloaderTest, download_virtual_settings)
" ]\n"
"}";
EXPECT_CALL(mock_communication, downloadAttributeFile(resourse_file)).WillOnce(Return(fog_response));
EXPECT_CALL(
mock_communication,
downloadAttributeFile(resourse_file, "/tmp/virtualSettings_general.download"))
.WillOnce(Return(fog_response));
EXPECT_CALL(
mock_orchestration_tools,
readFile("/tmp/virtualSettings_general.download")
).WillOnce(Return(fog_response));
stringstream tenant_0000_path;
tenant_0000_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789"

4
components/security_apps/orchestration/downloader/http_client.cc
View File

@@ -221,7 +221,7 @@ HTTPClient::curlGetFileOverHttp(const URLParser &url, ofstream &out_file, const
token,
proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
proxy_config->getProxyPort(ProxyProtocol::HTTPS),
proxy_config->getProxyCredentials(ProxyProtocol::HTTPS));
proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS));
http_curl_client.setCurlOpts();
bool connection_ok = http_curl_client.connect();
@@ -251,7 +251,7 @@ HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string &
url,
proxy_config->getProxyDomain(ProxyProtocol::HTTP),
proxy_config->getProxyPort(ProxyProtocol::HTTP),
proxy_config->getProxyCredentials(ProxyProtocol::HTTP),
proxy_config->getProxyAuthentication(ProxyProtocol::HTTP),
token
);
auto handle_connect_res = client_connection.handleConnect();

2
components/security_apps/orchestration/downloader/http_client.h
View File

@@ -17,14 +17,12 @@
#include <string>
#include "maybe_res.h"
#include "url_parser.h"
#include "i_messaging.h"
#include "i_agent_details.h"
#include "i_proxy_configuration.h"
// LCOV_EXCL_START Reason: Depends on real download server.
class HTTPClient
:
public Singleton::Consume<I_Messaging>,
public Singleton::Consume<I_AgentDetails>,
public Singleton::Consume<I_ProxyConfiguration>
{

4
components/security_apps/orchestration/downloader/https_client.cc
View File

@@ -544,7 +544,7 @@ HTTPClient::getFileSSL(const URLParser &url, ofstream &out_file, const string &t
url,
proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
proxy_config->getProxyPort(ProxyProtocol::HTTPS),
proxy_config->getProxyCredentials(ProxyProtocol::HTTPS),
proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS),
token
);
@@ -589,7 +589,7 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
token,
proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
proxy_config->getProxyPort(ProxyProtocol::HTTPS),
proxy_config->getProxyCredentials(ProxyProtocol::HTTPS),
proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS),
cert_file_path);
ssl_curl_client.setCurlOpts();