diff --git a/scripts/build_agent.sh b/scripts/build_agent.sh new file mode 100755 index 0000000..608ca35 --- /dev/null +++ b/scripts/build_agent.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +build_image() +{ + local image_name=$1 + local ver=$2 + echo "Build image: docker build -t openappsec-${image_name}:${ver} --network host --build-arg VERSION=${ver} -f scripts/${image_name}/Dockerfile ." + docker build -t openappsec-"${image_name}":"${ver}" --network host --build-arg VERSION="${ver}" -f "scripts/${image_name}"/Dockerfile . + local ret_code=$? + if [[ $ret_code != 0 ]]; then + echo "Build image: docker build failed with error $?" + exit $ret_code + fi +} + +build_agent() +{ + local flavor=$1 + local ver=$2 + CI_PROJECT_DIR=$PWD + echo "oriane" + echo "${CI_PROJECT_DIR}" + echo "ROY" + mkdir -p ${CI_PROJECT_DIR}/output/${flavor}/${ver} + echo "Run image: docker run --security-opt seccomp=unconfined -e linux_dist="${flavor}" -e dist_ver="${ver}" -e package=agent -v "${CI_PROJECT_DIR}"/output/${flavor}/${ver}/:/output --name openappsec-"${flavor}"-"${ver}"-c -i openappsec-"${flavor}":"${ver}" /build_artifacts.sh" + docker run --security-opt seccomp=unconfined -e linux_dist="${flavor}" -e dist_ver="${ver}" -e package=agent -v ${PWD}:/openappsec:rw -v "${CI_PROJECT_DIR}"/output/${flavor}/${ver}/:/output --name openappsec-"${flavor}"-"${ver}"-c -i openappsec-"${flavor}":"${ver}" /build_artifacts.sh + local ret_code=$? + if [[ $ret_code != 0 ]]; then + echo "Run image: docker run failed with error $?" + exit $ret_code + fi +} + +handle_flavor() +{ + local flavor=$1 + shift + local versions=("$@") + for ver in "${versions[@]}"; do + build_image $flavor $ver + build_agent $flavor $ver + done +} + +handle_flavor $1 $2 + diff --git a/scripts/build_artifacts.sh b/scripts/build_artifacts.sh new file mode 100755 index 0000000..bcc3f78 --- /dev/null +++ b/scripts/build_artifacts.sh @@ -0,0 +1,444 @@ +#!/bin/bash + +build_agent() +{ + #echo "Fetching source code for open appsec agent" + #git clone https://github.com/openappsec/openappsec.git + #if [[ $? != 0 ]]; then + # echo "Failed to clones source code for openappsec agent" + # exit 1 + #fi + cd openappsec + + if [ -z "$(find /usr -name libgtest.*)" ]; then + echo "Building gtest library" + if [[ "${linux_dist}" == "ubuntu" ]]; then + apt-get -qq install google-mock -y + if [[ $? != 0 ]]; then + echo "Failed to install google-mock" + exit 1 + fi + fi + cd /usr/src/googletest + cmake CMakeLists.txt + if [[ $? != 0 ]]; then + echo "Failed to run cmake as part of googletest compilation" + exit 1 + fi + + make + if [[ $? != 0 ]]; then + echo "Failed to compile google-mock" + exit 1 + fi + + cp /usr/src/googletest/googlemock/gtest/libgtest.a /usr/lib/libgtest.a + cp /usr/src/googletest/googlemock/gtest/libgtest_main.a /usr/lib/libgtest_main.a + cp /usr/src/googletest/googlemock/libgmock.a /usr/lib/libgmock.a + cd - + fi + + echo "Building libraries for open appsec agent" + local var_cmake_ret_code + if [[ "${linux_dist}" == "centos" ]]; then + echo "function(add_unit_test ut_name ut_sources use_libs)" > unit_test.cmake + echo "endfunction(add_unit_test)" >> unit_test.cmake + sed -i "s|add_subdirectory(cptest)|#add_subdirectory(cptest)|g" core/CMakeLists.txt + + cmake -DCMAKE_INSTALL_PREFIX=build_out -DBoost_LIBRARY_DIRS=/usr/lib64/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 -DBOOST_INCLUDEDIR=/usr/include/boost169/ -DCMAKE_CXX_FLAGS="-std=gnu++11 -I/usr/include/openssl11/ -L/usr/lib64/openssl11/" + var_cmake_ret_code=$? + elif [[ "${linux_dist}" == "rhel" && "${dist_ver}" == "8" ]]; then + cmake -DCMAKE_INSTALL_PREFIX=build_out -DBoost_LIBRARY_DIRS=/usr/lib64/boost169 -DBOOST_LIBRARYDIR=/usr/lib64/boost169 -DBOOST_INCLUDEDIR=/usr/include/boost169/ -DCMAKE_CXX_FLAGS="-L/usr/lib64/boost169/" + var_cmake_ret_code=$? + elif [[ "${linux_dist}" == "fedora" ]]; then + echo "function(add_unit_test ut_name ut_sources use_libs)" > unit_test.cmake + echo "endfunction(add_unit_test)" >> unit_test.cmake + sed -i "s|add_subdirectory(cptest)|#add_subdirectory(cptest)|g" core/CMakeLists.txt + cmake -DCMAKE_INSTALL_PREFIX=build_out . + var_cmake_ret_code=$? + else + cmake -DCMAKE_INSTALL_PREFIX=build_out . + var_cmake_ret_code=$? + fi + + if [[ $var_cmake_ret_code != 0 ]]; then + echo "Failed to run cmake on openappsec agent code" + exit 1 + fi + + local var_make_success=0 + for ((try=0; try<10; try++)); do + make -j 8 install && var_make_success=1 && break + done + + if [[ $var_make_success == 0 ]]; then + echo "Failed to run cmake on openappsec agent code" + exit 1 + fi + + echo "Building packages for open appsec agent" + make package + if [[ $? != 0 ]]; then + echo "Failed to package openappsec agent" + exit 1 + fi + + mkdir openappsec + cp build_out/install*.sh openappsec/ + + echo "Compressing packages for open appsec agent" + tar -czvf openappsec-${dist_ver}.tar.gz openappsec/ + if [[ $? != 0 ]]; then + echo "Failed to compress openappsec agent" + exit 1 + fi + + cd .. + echo "Saving packagestar for open appsec agent. Path: output/agent/openappsec-${dist_ver}.tar.gz" + mkdir -p "output/agent" + cp openappsec/openappsec-${dist_ver}.tar.gz "output/agent/" +} + +build_specific_attachment() +{ + local raw_nginx_ver="$1" + local nginx_ver="$(echo ${raw_nginx_ver} | sed "s|~|-|g" | sed "s| |-|g")" + cp -R "attachment_source" "attachment_${nginx_ver}" + if [[ $? != 0 ]]; then + echo "Failed to copy attachment source code, Error $?" + exit 1 + fi + cd "attachment_${nginx_ver}" + + echo "Building libraries for open appsec attachment" + + nginx -V &> /tmp/nginx.ver + rm -f /tmp/nginx.configure.output + sed -i -e 's|make|make -j 8|' ./attachments/nginx/ngx_module/nginx_version_configuration.sh + ./attachments/nginx/ngx_module/nginx_version_configuration.sh --conf /tmp/nginx.ver build_out >> /tmp/nginx.configure.output + if [[ $? != 0 ]]; then + echo "Failed to configure attachment source code" + cat /tmp/nginx.configure.output + rm -f /tmp/nginx.configure.output + exit 1 + fi + + sed -i "s|install(TARGETS ngx_module DESTINATION lib)|#install(TARGETS ngx_module DESTINATION lib)|g" docker/CMakeLists.txt + if [[ "${linux_dist}" == "centos" ]]; then + cmake -DCMAKE_INSTALL_PREFIX=build_out . -DCMAKE_CXX_FLAGS="-std=gnu++11 -I/usr/include/openssl11/ -L/usr/lib64/openssl11/" -DCMAKE_C_FLAGS="-std=gnu99 -I/usr/include/openssl11/ -L/usr/lib64/openssl11/" + if [[ $? != 0 ]]; then + echo "Failed to run cmake on attachment source code" + exit 1 + fi + else + cmake -DCMAKE_INSTALL_PREFIX=build_out . + if [[ $? != 0 ]]; then + echo "Failed to run cmake on attachment source code" + exit 1 + fi + fi + make -j 8 install + if [[ $? != 0 ]]; then + echo "Failed to run make on attachment source code" + exit 1 + fi + + mkdir "ngx_module_${nginx_ver}" + cp build_out/lib/lib*so "ngx_module_${nginx_ver}"/ + tar -czvf "ngx_module_${nginx_ver}.tar.gz" "ngx_module_${nginx_ver}"/ + if [[ $? != 0 ]]; then + echo "Failed to compress attachment artifacts" + exit 1 + fi + cd .. + + mkdir -p "output/attachment" + cp "attachment_${nginx_ver}"/"ngx_module_${nginx_ver}.tar.gz" "output/attachment" + rm -rf "attachment_${nginx_ver}" + + if [[ "${linux_dist}" == "rhel" ]]; then + /usr/bin/yum remove -y "nginx-${raw_nginx_ver}" + fi +} + +nginx_version_comparator() +{ + local ver1_number + local ver2_number + + IFS='.' read -r -a ver1_numbers <<< "$1" + IFS='.' read -r -a ver2_numbers <<< "$2" + + local IFS=. + local len1=${#ver1_numbers[@]} + local len2=${#ver2_numbers[@]} + max=$(( len1 > len2 ? len1 : len2 )) + for ((i=0; i /tmp/nginx.ver + rm -f /tmp/nginx.configure.output + sed -i -e 's|make|make -j 8|' ./attachments/nginx/ngx_module/nginx_version_configuration.sh + ./attachments/nginx/ngx_module/nginx_version_configuration.sh --conf /tmp/nginx.ver build_out >> /tmp/nginx.configure.output + if [[ $? != 0 ]]; then + echo "Failed to configure attachment source code" + cat /tmp/nginx.configure.output + rm -f /tmp/nginx.configure.output + exit 1 + fi + + sed -i "s|install(TARGETS ngx_module DESTINATION lib)|#install(TARGETS ngx_module DESTINATION lib)|g" docker/CMakeLists.txt + if [[ "${linux_dist}" == "centos" ]]; then + cmake -DCMAKE_INSTALL_PREFIX=build_out . -DCMAKE_CXX_FLAGS="-std=gnu++11 -I/usr/include/openssl11/ -L/usr/lib64/openssl11/" -DCMAKE_C_FLAGS="-std=gnu99 -I/usr/include/openssl11/ -L/usr/lib64/openssl11/" + if [[ $? != 0 ]]; then + echo "Failed to run cmake on attachment source code" + exit 1 + fi + else + cmake -DCMAKE_INSTALL_PREFIX=build_out . + if [[ $? != 0 ]]; then + echo "Failed to run cmake on attachment source code" + exit 1 + fi + fi + make -j 8 install + if [[ $? != 0 ]]; then + echo "Failed to run make on attachment source code" + exit 1 + fi + + mkdir "openresty_${openresty_ver}" + cp build_out/lib/lib*so "openresty_${openresty_ver}"/ + tar -czvf "openresty_${openresty_ver}.tar.gz" "openresty_${openresty_ver}"/ + if [[ $? != 0 ]]; then + echo "Failed to compress attachment artifacts" + exit 1 + fi + cd .. + + mkdir -p "output/attachment" + cp "attachment_${openresty_ver}"/"openresty_${openresty_ver}.tar.gz" "output/attachment" + rm -rf "attachment_${openresty_ver}" +} + +install_kong() +{ + local ver=$1 + local linux_flavor=$2 + local dist_version=$3 + + echo "Installing kong for os: $linux_flavor, kong ver: $ver" + if [[ "$linux_flavor" == "ubuntu" || "$linux_flavor" == "debian" ]]; then + echo "echo \"$ver\" | cut -d"_" -f2 | cut -d"." -f1" + local gateway_version=$(echo "$ver" | cut -d"_" -f2 | cut -d"." -f1) + echo "curl -LO \"https://download.konghq.com/gateway-$gateway_version.x-$linux_flavor-$dist_version/pool/all/k/kong-enterprise-edition/$ver\"" + curl -LO "https://download.konghq.com/gateway-$gateway_version.x-$linux_flavor-$dist_version/pool/all/k/kong-enterprise-edition/$ver" + echo "dpkg -i --force-confnew \"$ver\"" + dpkg -i --force-confnew "$ver" + elif [[ "$linux_flavor" == "centos" ]]; then + echo "echo \"$ver\" | cut -d"-" -f4 | cut -d"." -f1" + local gateway_version=$(echo "$ver" | cut -d"-" -f4 | cut -d"." -f1) + echo "curl -LO \"https://download.konghq.com/gateway-$gateway_version.x-$linux_flavor-$dist_version/Packages/k/$ver\"" + curl -LO "https://download.konghq.com/gateway-$gateway_version.x-$linux_flavor-$dist_version/Packages/k/$ver" + echo "yum -y install \"$ver\"" + yum -y install "$ver" + fi + + if [[ $? == 0 ]]; then + return 0 + fi + return 1 +} + +build_kong() +{ + local kong_type=$1 + local kong_version_path=$2 + + echo "Fetching source code for open appsec attachment" + git clone https://github.com/openappsec/attachment.git "attachment_source" + if [[ $? != 0 ]]; then + echo "Failed to clone attachment source code" + exit 1 + fi + + echo "Starting iteration over $kong_type list of distroes" + local kong_versions_to_install="$kong_version_path" + while IFS= read -r line; do + echo "*********** Installing $kong_type ver=$line os=$linux_dist ***********" + if [[ "$linux_dist" == "ubuntu" || "$linux_dist" == "debian" ]]; then + echo "/usr/bin/apt-get install -y $kong_type=$line" + /usr/bin/apt-get install -y $kong_type=$line + elif [[ "$linux_dist" == "centos" ]]; then + echo "/usr/bin/yum install -y $kong_type-$line" + /usr/bin/yum install -y $kong_type-$line + fi + + if [[ $? == 0 ]]; then + NGINX_VERSION=$(/usr/local/openresty/nginx/sbin/nginx -v |& cut -d/ -f2) + build_specific_kong_attachment "$NGINX_VERSION" + else + echo "" + echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + echo "+++++ ERROR: $kong_type version isn't supported from the package manager +++++" + echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" + echo "" + fi + + if [[ "$linux_dist" == "ubuntu" || "$linux_dist" == "debian" ]]; then + /usr/bin/apt-get remove -y $kong_type + elif [[ "$linux_dist" == "centos" ]]; then + /usr/bin/yum remove -y $kong_type + fi + done < "$kong_versions_to_install" + rm -rf "attachment_source" +} + +if [[ "${package}" == "agent" ]]; then + build_agent +elif [[ "${package}" == "attachments" ]]; then + build_attachments +elif [[ "${package}" == "kong" ]]; then + build_kong kong KONG.list + build_kong kong-enterprise-edition KONG-ENTERPRISE.list +else + echo "Failed to pick agent/attachments/kong to build" +fi + diff --git a/scripts/entry.sh b/scripts/entry.sh new file mode 100755 index 0000000..23172a2 --- /dev/null +++ b/scripts/entry.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +build() +{ + local part=$1 + local flavor=$2 + local version="$3" + + echo "Building something with $part $flavor $version" + if [[ $1 == "agent" ]]; then + echo "Building agent for $2 $3" + $PWD/scripts/build_agent.sh $flavor $version + fi +} + +publish_output() +{ + bash ./scripts/generate_openappsec_versions_report.sh + local ret_code=$? + if [[ $ret_code != 0 ]]; then + echo "publish image: generate version report failed with error $?" + fi + exit $ret_code +} + +make_latest() +{ + /ngen/builds/agent-build/master/latest/scripts/latest.py + local ret_code=$? + if [[ $ret_code != 0 ]]; then + echo "Make latest failed with error $?" + fi + exit $ret_code +} + +main() +{ + docker rm `docker ps -aq` + mkdir -p $PWD/output + + if [[ "$2" == "amzn" || "$2" == "opensuse" || "$2" == "alpine" || "$2" == "rhel" || "$2" == "fedora" ]]; then + echo "$2 flavor is currently unsupported" + exit 0 + fi + + local platform_target="$2" + + build "$1" "$2" "$3" + +# if [[ $(find output/${platform_target} -type f -size -1k | wc -l) -ne 0 ]]; then +# echo "Error! The following artifacts are malformed:" +# find output/${platform_target} -type f -size -1k +# exit 1 +# fi + +# if [[ "${CI_BUILD_REF_NAME}" == "dev" || "${CI_BUILD_REF_NAME}" == "master" || "${CI_BUILD_REF_NAME}" == "release" || "${CI_BUILD_REF_NAME}" == "hotfix-*" ]]; then +# find output/${platform_target} -name "openappsec-*" | awk -F/ '{system("mkdir -p "$1"/agent/x86_64/"$2"/"$3"; cp "$0" "$1"/agent/x86_64/"$2"/"$3"/"$5)}' +# find output/agent/x86_64/${platform_target} -maxdepth 1 -mindepth 1 | sed 's/output.//' | xargs -IXXX /ngen/builds/agent-build/master/latest/scripts/export.py --platform XXX output +# fi +} + +main $1 $2 $3 diff --git a/scripts/ubuntu/Dockerfile b/scripts/ubuntu/Dockerfile new file mode 100644 index 0000000..06f864e --- /dev/null +++ b/scripts/ubuntu/Dockerfile @@ -0,0 +1,11 @@ +ARG VERSION +FROM ubuntu:$VERSION + +RUN mkdir -p output/ +COPY scripts/ / +RUN apt-get update +RUN DEBIAN_FRONTEND=noninteractive apt-get install -y libboost-all-dev libssl-dev libpcre2-dev libxml2-dev libgtest-dev libgmock-dev libcurl4-gnutls-dev +RUN apt-get install -y cmake build-essential +RUN chmod +x build_artifacts.sh +ENTRYPOINT /build_artifacts.sh +