diff --git a/CMakeLists.txt b/CMakeLists.txt
index 060a316..cc2f8e9 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,12 +1,13 @@
-cmake_minimum_required (VERSION 2.8.4)
-project (ngen)
+cmake_minimum_required(VERSION 2.8.4)
+project(ngen)
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wno-terminate")
-set (CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 11)
include_directories(external)
include_directories(core/include/attachments)
add_subdirectory(core)
add_subdirectory(attachments)
+add_subdirectory(docker)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index b9540ab..43f125c 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,7 +1,8 @@
# Welcome to open-appsec's Open Sources Contributing Guide
-We welcome everyone that wishes to share their expetise in improving open-appsec.
+Thank you for investing your time to consider a contribution to our project!
+Various open source repositories were released as part of Check Point's platform for distributing and deploying security applications for a variety of environments. We welcome everyone that wishes to share their knowledge and expertise to enhance and expand both our platform, and our ability to secure more environments.
-Please ead our [Code of Conduct](./CODE_OF_CONDUCT.md) to keep our community approachable and respectable.
+Read our [Code of Conduct](./CODE_OF_CONDUCT.md) to keep our community approachable and respectable.
In this guide we will provide an overview of the various contribution options' guidelines - from reporting or fixing a bug, to suggesting an enhancement.
@@ -15,7 +16,7 @@ An internal process will be activated upon determining the validity of a reporte
**Important - If the bug you wish to report regards a suspicion of a security vulnerability, please refer to the "Reporting security vulnerability" section**
-To report a bug, you can either open a new issue using a relevant [issue form](https://github.com/github/docs/issues/new/choose), or, alternatively, contact us using [this email](mailto:opensource@openappsec.io).
+To report a bug, you can either open a new issue using a relevant [issue form](https://github.com/github/docs/issues/new/choose), or, alternatively, [contact us via our open-appsec open source distribution list](mailto:opensource@openappsec.io).
Be sure to include a **title and clear description**, as much relevant information as possible, and a **code sample** or an **executable test case** demonstrating the expected behavior that is not occurring.
@@ -30,3 +31,10 @@ Please [suggest your change via our open-appsec open source distribution list](m
## Open Source documentation issues
For reporting or suggesting a change, of any issue detected in the documentation files of our open source repositories, please use the same guidelines as bug reports/fixes.
+
+# Final Thanks
+We value all users that use our open source files.
+We value all efforts to read, suggest changes and/or contribute to our open source files.
+Thank you for it.
+
+The open-appsec Team
diff --git a/README.md b/README.md
index d7383c5..87b9d53 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,11 @@
-
openappsec/attachment
+
openappsec/attachement
## About
-open-appsec (https://www.openappsec.io) is a machine learning security engine that preemptively and automatically prevent threats against Web Application & APIs.
+open-appsec is a machine learning security engine that preemptively and automatically prevent threats against Web Application & APIs.
open-appsec Attachments connect between processes that provide HTTP data and the open-appsec Agent security logic.
@@ -18,44 +18,78 @@ This repository will host Attachment for different platforms. The first one is t
## open-appsec NGINX attachment compilation instructions
-### Compiling the attachment code
+The attchment can be compiled to support an existing alpine based nginx server or an nginx/ingress-nginx alpine based docker.
+
+Your compilation environment must be alpine based and contain git, docker, cmake and g++.
+
+Before compiling, ensure the latest development versions of the following libraries:
+
+* PCRE
+* libxml2
+* zlib
+* OpenSSL
+* Geoip
+
+```bash
+ $ apk update
+ $ apk add pcre-dev libxml2-dev zlib-dev openssl-dev geoip-dev
+```
+
+### Compiling the attachment code for an existing nginx server
+
+On your existing nginx server:
+1. Run command to extract nginx compilation flags to a file
+
+```bash
+ $ nginx -V &> /tmp/nginx.ver
+```
+
+On your compilation environment:
1. Clone this repository
-2. Run CMake command
-3. Run make install command
+2. Copy the file created on your nginx server (the previous section) to your compilation environment to the path /tmp/nginx.ver
+3. Run Configuration script
+4. Run CMake command
+5. Run make command
```bash
$ git clone https://github.com/openappsec/attachment.git
- $ cd attachment/
+ $ ./attachments/nginx/ngx_module/nginx_version_configuration.sh --conf /tmp/nginx.ver build_out
$ cmake -DCMAKE_INSTALL_PREFIX=build_out .
$ make install
```
-### NGINX plugin
-
-NGINX Plugins are built per specific version.
-1. Get nginx source code from [nginx.org](http://nginx.org/), e.g. version 1.23.0 (see [nginx compatibility](http://nginx.org/en/docs/njs/compatibility.html))
-2. Run make modules
-
-```bash
- $ module_path=//attachment
-
- $ wget 'https://nginx.org/download/nginx-1.23.0.tar.gz'
- $ sha256sum nginx-1.23.0.tar.gz
- 820acaa35b9272be9e9e72f6defa4a5f2921824709f8aa4772c78ab31ed94cd1 nginx-1.23.0.tar.gz
-
- $ tar -xzvf nginx-1.23.0.tar.gz
- $ cd nginx-1.23.0/
-
- $ ./configure --add-dynamic-module=$module_path --with-cc-opt="-I $module_path/core/include/attachments"
-
- $ make modules
-```
-
#### NGINX plugin associated libraries
The NGINX plugin uses these libraries: shmem_ipc, compression_utils, and nginx_attachment_util.
They can be found under the `lib` directory in the `` given to the CMake.
+#### Deploying the attachment on an existing alpine nginx server
+
+1. Copy the associated libraries to /usr/lib on your existing nginx server
+2. Copy the nginx attachment file lib/libngx_module.so to the following path on your existing nginx server: /usr/lib/nginx/modules/
+3. Load the attachment on your nginx by adding the following command to the main nginx.conf file:
+ load_module /usr/lib/nginx/modules/libngx_module.so;
+4. Restart your nginx server.
+
+### Compiling the attachment code and create a docker image for an existing nginx/ingres-nginx alpine docker
+
+This step requires Docker to be installed on your comilation environment
+
+1. Clone this repository
+3. Run Configuration script with the required docker image name and tag
+4. Run CMake command
+5. Run make command
+
+```bash
+ $ git clone https://github.com/openappsec/attachment.git
+ $ ./attachments/nginx/ngx_module/nginx_version_configuration.sh --docker .
+ $ make install
+ $ make docker
+```
+
+Later on, you can push the image to your own registry in use it as needed.
+
## License
open-appsec/attachment is open source and available under the Apache 2.0 license.
diff --git a/attachments/nginx/CMakeLists.txt b/attachments/nginx/CMakeLists.txt
index ef815f9..196e407 100644
--- a/attachments/nginx/CMakeLists.txt
+++ b/attachments/nginx/CMakeLists.txt
@@ -1 +1,2 @@
add_subdirectory(nginx_attachment_util)
+add_subdirectory(ngx_module)
diff --git a/attachments/nginx/nginx_attachment_util/CMakeLists.txt b/attachments/nginx/nginx_attachment_util/CMakeLists.txt
index ba3aa76..5a8e158 100644
--- a/attachments/nginx/nginx_attachment_util/CMakeLists.txt
+++ b/attachments/nginx/nginx_attachment_util/CMakeLists.txt
@@ -1,5 +1,6 @@
add_definitions(-DUSERSPACE)
add_library(osrc_nginx_attachment_util SHARED nginx_attachment_util.cc)
+target_link_libraries(osrc_nginx_attachment_util http_configuration)
install(TARGETS osrc_nginx_attachment_util DESTINATION lib)
diff --git a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc
index df2edcb..32e66cd 100644
--- a/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc
+++ b/attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc
@@ -83,6 +83,18 @@ getFailOpenTimeout()
return conf_data.getNumericalValue("fail_open_timeout");
}
+int
+isFailOpenHoldMode()
+{
+ return conf_data.getNumericalValue("is_fail_open_mode_hold_enabled");
+}
+
+unsigned int
+getFailOpenHoldTimeout()
+{
+ return conf_data.getNumericalValue("fail_open_hold_timeout");
+}
+
unsigned int
getMaxSessionsPerMinute()
{
@@ -137,6 +149,12 @@ getResBodyThreadTimeout()
return conf_data.getNumericalValue("res_body_thread_timeout_msec");
}
+unsigned int
+getWaitingForVerdictThreadTimeout()
+{
+ return conf_data.getNumericalValue("waiting_for_verdict_thread_timeout_msec");
+}
+
int
isIPAddress(c_str ip_str)
{
diff --git a/attachments/nginx/ngx_module/CMakeLists.txt b/attachments/nginx/ngx_module/CMakeLists.txt
new file mode 100644
index 0000000..5b08a97
--- /dev/null
+++ b/attachments/nginx/ngx_module/CMakeLists.txt
@@ -0,0 +1,48 @@
+MACRO(CREATE_INCLUDE_LIST result)
+ file(READ ${CMAKE_INSTALL_PREFIX}/nginx-src/include_paths.mk relative_includes)
+ STRING(REGEX REPLACE "\n" ";" relative_includes "${relative_includes}")
+ set(inclist "")
+ FOREACH(include ${relative_includes})
+ if(NOT include MATCHES "^/")
+ set(inclist ${inclist} "${CMAKE_INSTALL_PREFIX}/nginx-src/${include}")
+ else()
+ set(inclist ${inclist} "${include}")
+ endif()
+ ENDFOREACH()
+ set(${result} ${inclist})
+ENDMACRO()
+
+MACRO(READ_COMPILE_FLAGS result)
+ file(READ ${CMAKE_INSTALL_PREFIX}/nginx-src/cc_flags.mk CC_FLAGS)
+ string(REGEX REPLACE "\n" "" CC_FLAGS "${CC_FLAGS}")
+ set(flag_list "")
+ FOREACH(flag ${CC_FLAGS})
+ if (flag MATCHES "-fstack-clash-protection" OR flag MATCHES "-fcf-protection" OR flag MATCHES "-Wno-cast-function-type")
+ continue()
+ endif()
+
+ set(flag_list ${flag_list} "${flag}")
+ ENDFOREACH()
+ separate_arguments(flag_list)
+ set(${result} ${flag_list})
+ENDMACRO()
+
+add_library(
+ ngx_module
+ SHARED
+ ngx_http_cp_attachment_module.c ngx_cp_thread.c ngx_cp_hook_threads.c ngx_cp_hooks.c ngx_cp_utils.c
+ ngx_cp_initializer.c ngx_cp_io.c ngx_cp_static_content.c ngx_cp_custom_response.c ngx_modules.c
+ ngx_cp_compression.c ngx_cp_http_parser.c ngx_cp_failing_state.c ngx_cp_metric.c
+)
+
+add_dependencies(ngx_module osrc_shmem_ipc osrc_nginx_attachment_util osrc_compression_utils)
+
+target_link_libraries(ngx_module osrc_shmem_ipc osrc_nginx_attachment_util osrc_compression_utils)
+
+CREATE_INCLUDE_LIST(NGX_INCLUDES)
+READ_COMPILE_FLAGS(CC_FLAG_LIST)
+
+target_include_directories(ngx_module PRIVATE ${NGX_INCLUDES})
+target_compile_options(ngx_module PRIVATE ${CC_FLAG_LIST})
+
+install(TARGETS ngx_module DESTINATION lib)
diff --git a/attachments/nginx/ngx_module/nginx_version_configuration.sh b/attachments/nginx/ngx_module/nginx_version_configuration.sh
new file mode 100755
index 0000000..0db4951
--- /dev/null
+++ b/attachments/nginx/ngx_module/nginx_version_configuration.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+NGINX_VERSION_CONF_INPUT_PATH=/tmp/nginx.sourcefile.ver
+if [ "${1}" == "--docker" ]; then
+ docker run -it ${2} nginx -V > ${NGINX_VERSION_CONF_INPUT_PATH}
+elif [ "${1}" == "--conf" ]; then
+ cp ${2} ${NGINX_VERSION_CONF_INPUT_PATH}
+else
+ echo "Usage: ${0} [--conf "
+ exit 1
+fi
+NGINX_VERSION_CONF_OUTPUT_PATH=/tmp/nginx.sourcefile.conf
+$(dirname $0)/nginx_version_extractor.sh -i ${NGINX_VERSION_CONF_INPUT_PATH} -o ${NGINX_VERSION_CONF_OUTPUT_PATH}
+BUILD_OUTPUT_DIR=${3}
+
+if [[ ${BUILD_OUTPUT_DIR} != /* ]]; then
+ BUILD_OUTPUT_DIR=$(pwd)/${BUILD_OUTPUT_DIR}
+fi
+
+source ${NGINX_VERSION_CONF_OUTPUT_PATH}
+
+CURRENT_PWD=$(pwd)
+
+mkdir -p ${BUILD_OUTPUT_DIR}
+cd ${BUILD_OUTPUT_DIR}
+
+wget --no-check-certificate https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz
+
+tar -xzvf nginx-${NGINX_VERSION}.tar.gz
+
+rm nginx-${NGINX_VERSION}.tar.gz
+
+mv nginx-${NGINX_VERSION} nginx-src
+cd nginx-src
+
+if test ! -f configured.ok; then
+ echo "Configuring nginx compiler: ./configure ${CONFIGURE_OPT} --with-cc-opt=\"${EXTRA_CC_OPT} ${LOCAL_CC_OPT}\" && touch configured.ok"
+ ./configure ${CONFIGURE_OPT} --with-cc-opt="${EXTRA_CC_OPT} ${LOCAL_CC_OPT}" && touch configured.ok
+else
+ echo "Nginx compiler already Configured...\n"
+fi
+
+make && echo "${EXTRA_CC_OPT}" > cc_flags.mk
+
+
+OUTPUT_FILE_NAME=include_paths.mk
+
+ALL_INCS=$(cat objs/Makefile|grep -v \$\(ALL_INCS\) | awk '/ALL_INCS/' RS="\n\n" ORS="\n\n" | tr '\\' ' ')
+ALL_INCS=${ALL_INCS/ALL_INCS =/}
+ALL_INCS=$(sed "s/-I//g" <<< "${ALL_INCS}")
+
+echo > include_paths.mk
+for include in ${ALL_INCS}; do
+ echo $include >> include_paths.mk
+done
+
+cd ${CURRENT_PWD}
+
+if [ "${1}" == "--docker" ]; then
+ cp -f $(dirname $0)/../../../docker/Dockerfile ${BUILD_OUTPUT_DIR}/Dockerfile
+ sed -i "s||${2}|g" ${BUILD_OUTPUT_DIR}/Dockerfile
+ docker run -it ${2} whoami > /tmp/usertouse
+ USER_NAME="$(cat /tmp/usertouse)"
+ rm /tmp/usertouse
+ sed -i "s||${USER_NAME}|g" ${BUILD_OUTPUT_DIR}/Dockerfile
+fi
diff --git a/attachments/nginx/ngx_module/nginx_version_extractor.sh b/attachments/nginx/ngx_module/nginx_version_extractor.sh
new file mode 100755
index 0000000..9f19850
--- /dev/null
+++ b/attachments/nginx/ngx_module/nginx_version_extractor.sh
@@ -0,0 +1,347 @@
+#!/bin/bash
+
+initializeEnviroment()
+{
+ CURRENT_TIME=""
+ PACKAGE_VERSION=""
+ CUR_NGINX_ALREADY_SUPPORTED=false
+ NUMBER_OF_CONFIGURATION_FLAGS=0
+ TMP_NGINX_UNPARSED_CONFIGURATION="/tmp/nginx_unparsed_tmp_conf.txt"
+ TMP_NGINX_PARSED_CONFIGURATION_FLAGS="/tmp/nginx_parsed_conf_flags.txt"
+ TMP_DECODED_FILE_PATH="/tmp/decoded_file.txt"
+ IS_ALPINE=false
+ if [ ! -z "$(cat /etc/*release | grep alpine)" ]; then
+ IS_ALPINE=true
+ fi
+}
+
+usage()
+{
+ local IS_ERROR=$1
+ local option=$2
+ if [[ ${IS_ERROR} == true ]]; then
+ echo "Error: unsupported option '${option}'"
+ fi
+
+ echo "Usage:"
+ line_padding=" "
+ local debug_print_option="-h, --help"
+ printf "%s %s Print (this) help message\n" "$debug_print_option" "${line_padding:${#debug_print_option}}"
+ debug_print_option="-d, --debug"
+ printf "%s %s Enable debug mode\n" "$debug_print_option" "${line_padding:${#debug_print_option}}"
+ debug_print_option="-v, --verbose"
+ printf "%s %s show version and configure options\n" "$debug_print_option" "${line_padding:${#debug_print_option}}"
+ debug_print_option="-o, --output"
+ printf "%s %s change output file name into '${option}'\n" "$debug_print_option" "${line_padding:${#debug_print_option}}"
+ debug_print_option="-f, --force"
+ printf "%s %s force creation of makefile'\n" "$debug_print_option" "${line_padding:${#debug_print_option}}"
+
+ if [[ ${IS_ERROR} == true ]]; then
+ exit -1
+ else
+ exit 1
+ fi
+}
+
+debug()
+{
+ local debug_message=$1
+ if [[ $IS_DEBUG_MODE_ACTIVE == true ]]; then
+ echo -e $debug_message
+ fi
+}
+
+check_flags_options()
+{
+ local argc=$#
+
+ for (( i = 1; i <= $argc; i++ )); do
+ local option=${!i}
+ local IS_ERROR=false
+ if [[ "$option" == "--debug" || "$option" == "-d" ]]; then
+ IS_DEBUG_MODE_ACTIVE=true
+ elif [[ "$option" == "--verbose" || "$option" == "-v" ]]; then
+ IS_VERBOSE_MODE_ACTIVE=true
+ elif [[ "$option" == "--force" || "$option" == "-f" ]]; then
+ IS_FORCE_OUTPUT=true
+ elif [[ "$option" == "--output" || "$option" == "-o" ]]; then
+ IS_OUTPUT_NAME_MODE_ACTIVE=true
+ i=$((i+1))
+ FILE_NAME=${!i}
+ if [[ -z ${FILE_NAME} ]]; then
+ echo "Error: No file name was given for ${option} option."
+ exit -1
+ fi
+ elif [[ "$option" == "--input" || "$option" == "-i" ]]; then
+ i=$((i+1))
+ if [[ -z ${!i} ]]; then
+ echo "Error: No file name was given for ${option} option."
+ exit -1
+ fi
+ cp -f ${!i} ${TMP_NGINX_UNPARSED_CONFIGURATION}
+ IS_INPUT_NAME_MODE_ACTIVE=true
+ elif [[ "$option" == "--help" || "$option" == "-h" ]]; then
+ usage ${IS_ERROR} ${option}
+ elif [[ ! -z $option ]]; then
+ IS_ERROR=true
+ usage ${IS_ERROR} ${option}
+ fi
+ done
+}
+
+_main()
+{
+ echo "Starting verification of Check Point support with local nginx server"
+ initializeEnviroment
+
+ if [[ ${IS_INPUT_NAME_MODE_ACTIVE} != true ]]; then
+ nginx -V &> "$TMP_NGINX_UNPARSED_CONFIGURATION"
+ fi
+ getNginxVersion
+
+ if [[ $IS_VERBOSE_MODE_ACTIVE == true ]]; then
+ echo ""
+ cat ${TMP_NGINX_UNPARSED_CONFIGURATION}
+ echo ""
+ fi
+
+ while IFS= read -ra UNPARSED_CONFIGURATION_LINE <&3; do
+ if [[ ${UNPARSED_CONFIGURATION_LINE} =~ ^"nginx version:" ]]; then
+ openFile
+ elif [[ ${UNPARSED_CONFIGURATION_LINE} =~ ^"built by gcc" ]]; then
+ addBuiltConfiguration "${UNPARSED_CONFIGURATION_LINE}"
+ elif [[ ${UNPARSED_CONFIGURATION_LINE} =~ ^"configure arguments:" ]]; then
+ IFS="'"
+ addAndCutOptionalFlags ${UNPARSED_CONFIGURATION_LINE}
+ IFS=" "
+ addRequiredFlags ${CONFIGURATION_FLAGES_NEED_TO_BE_PARSED}
+ fi
+ done 3<"$TMP_NGINX_UNPARSED_CONFIGURATION"
+
+ PARSED_CONFIGURATION="CONFIGURE_OPT=\"${COMBINED_CONFIGURATION_FLAGS}\"\n\n"
+ NUMBER_OF_CONFIGURATION_FLAGS=$((NUMBER_OF_CONFIGURATION_FLAGS-1))
+ local local_pwd=$(pwd)
+ if [[ ${local_pwd:0:2} == "//" ]]; then
+ local_pwd=${local_pwd:1}
+ fi
+ debug "Moving parsed configuration to target ${local_pwd}/${FILE_NAME} configuration file"
+ echo -e ${PARSED_CONFIGURATION} > ${FILE_NAME}
+ echo -e ${CC_OPTIONAL_FLAGS} >> ${FILE_NAME}
+ add_nginx_and_release_versions
+ if [[ $CUR_NGINX_ALREADY_SUPPORTED == true ]]; then
+ tearDown
+ echo -e "Check Point Nano Agent already supported on this environment"
+ else
+ tearDown
+ echo -e "Extracted environment data to $(pwd)/${FILE_NAME} \nPlease send file to nano-agent-attachments-support@checkpoint.com"
+ fi
+}
+
+tearDown()
+{
+ rm -f ${TMP_NGINX_UNPARSED_CONFIGURATION}
+ rm -f ${TMP_NGINX_PARSED_CONFIGURATION_FLAGS}
+ rm -f ${TMP_DECODED_FILE_PATH}
+ rm -f ${TMP_NGINX_VERSION_FILE}
+}
+
+getNginxVersion()
+{
+ TMP_NGINX_VERSION_FILE="/tmp/nginx_version_file.txt"
+ cat ${TMP_NGINX_UNPARSED_CONFIGURATION} | grep "nginx version:" &> "$TMP_NGINX_VERSION_FILE"
+ if [[ $IS_ALPINE == true ]]; then
+ NGINX_VERSION=`cat ${TMP_NGINX_VERSION_FILE} | grep -oE [0-9]+.[0-9]+.[0-9]+`
+ else
+ NGINX_VERSION=`cat ${TMP_NGINX_VERSION_FILE} | grep -oP [0-9]+.[0-9]+.[0-9]+`
+ fi
+}
+
+openFile()
+{
+ if [[ ${IS_OUTPUT_NAME_MODE_ACTIVE} != true ]]; then
+ FILE_NAME="${NGINX_VERSION}.mk"
+ debug "Trying to create an empty ${NGINX_VERSION} file"
+ FILE_NAME_PATH="$(pwd)/${FILE_NAME}"
+
+ if [[ -z ${FILE_NAME_PATH} || ! ( ${FILE_NAME} =~ [0-9]+.[0-9]+.[0-9]+.mk ) ]]; then
+ echo "ERROR: can't find nginx version."
+ exit -1
+ fi
+
+ if [[ -f "${FILE_NAME_PATH}" ]]; then
+ echo "The output file: ${FILE_NAME} already exists. Do you want to overwrite this file? [y/N]"
+ read answer
+ if [[ ${answer} != "y" ]]; then
+ echo -e "Stopping after the operation was cancelled.\nIf you wish to use other output file name you can use option -o or --output"
+ exit -1
+ fi
+ fi
+ else
+ debug "Trying to create an empty ${FILE_NAME} file"
+ FILE_NAME_PATH="${FILE_NAME}"
+ fi
+
+ touch ${FILE_NAME_PATH} &> /dev/null
+ if [ ! -e ${FILE_NAME_PATH} ];then
+ echo "Failed to create ${FILE_NAME_PATH}"
+ exit -1
+ fi
+ debug "Created an empty ${FILE_NAME} file"
+}
+
+checkAllDBLineFlags()
+{
+ local argc=$#
+ local argv=("$@")
+ local number_of_db_line_flags=$((argc-3))
+ local gcc_version_prefix="--with-cc="
+
+ if [[ ${number_of_db_line_flags} == ${NUMBER_OF_CONFIGURATION_FLAGS} ]]; then
+ for ((i = 3; i < ${argc}; i++)); do
+ if [[ ${argv[i]} =~ ^"${gcc_version_prefix}"* ]]; then
+ continue
+ fi
+ checkFlag ${argv[i]}
+ if [[ ${found_equal_flag} == false ]]; then
+ EQUAL_FLAGS=false
+ return
+ fi
+ done
+ else return
+ fi
+
+ EQUAL_FLAGS=true
+}
+
+checkFlag()
+{
+ found_equal_flag=false
+ db_flag=$1
+ while IFS='\' read -ra flag; do
+ if [[ "${flag}" == "${db_flag}" ]] || [[ "${flag} " == "${db_flag}" ]]; then
+ found_equal_flag=true
+ break
+ fi
+ done < ${TMP_NGINX_PARSED_CONFIGURATION_FLAGS}
+}
+
+addBuiltConfiguration()
+{
+ BUILT_BY_GCC_FLAG_PREFIX="--with-cc=/usr/bin/"
+ if [[ $IS_ALPINE == true ]]; then
+ GCC_VERSION=`echo "$1" | grep -oE "gcc "[0-9]+ | tr ' ' '-'`
+ else
+ GCC_VERSION=`echo "$1" | grep -oP "gcc "[0-9]+ | tr ' ' '-'`
+ fi
+ if [[ "$GCC_VERSION" == "gcc-4" ]]; then
+ GCC_VERSION=gcc-5
+ elif [[ "$GCC_VERSION" == "gcc-10" ]] || [[ "$GCC_VERSION" == "gcc-11" ]]; then
+ GCC_VERSION=gcc-8
+ fi
+ BUILT_BY_GCC_FLAG=" \\\\\n${BUILT_BY_GCC_FLAG_PREFIX}${GCC_VERSION}"
+ NUMBER_OF_CONFIGURATION_FLAGS=$((NUMBER_OF_CONFIGURATION_FLAGS+1))
+}
+
+addAndCutOptionalFlags()
+{
+ debug "Parsing all nginx configuration flags"
+ CC_EXTRA_PREFIX="EXTRA_CC_OPT="
+ CC_OPTIONAL_FLAG_PREFIX="--with-cc-opt="
+ LD_OPTIONAL_FLAG_PREFIX="--with-ld-opt="
+ local argc=$#
+ local argv=("$@")
+ for (( i = 0; i < $argc; i++ )); do
+ if [[ ${argv[i]} == *"${CC_OPTIONAL_FLAG_PREFIX}"* ]]; then
+ debug "Successfully added compilation flags"
+ CONFIGURATION_FLAGES_NEED_TO_BE_PARSED="${CONFIGURATION_FLAGES_NEED_TO_BE_PARSED}${argv[i]}"
+ i=$((i+1))
+ IFS=" "
+ addCCFlagsWithoutSpecsLocalFlag ${argv[i]}
+ CC_OPTIONAL_FLAGS="${CC_EXTRA_PREFIX}\"${CC_OPTIONAL_FLAGS}\""
+ elif [[ ${argv[i]} == *"${LD_OPTIONAL_FLAG_PREFIX}"* ]]; then
+ CONFIGURATION_FLAGES_NEED_TO_BE_PARSED="${CONFIGURATION_FLAGES_NEED_TO_BE_PARSED}${argv[i]}"
+ i=$((i+1))
+ else
+ CONFIGURATION_FLAGES_NEED_TO_BE_PARSED="${CONFIGURATION_FLAGES_NEED_TO_BE_PARSED}${argv[i]}"
+ fi
+ done
+ debug "Successfully finished adding optional flags"
+ }
+
+addCCFlagsWithoutSpecsLocalFlag()
+{
+ local argc=$#
+ local argv=("$@")
+ SPECS_FLAG_PREFIX="-specs="
+ NO_ERROR_PREFIX="-Wno-error="
+ FCF_PROTECTION_PREFIX="-fcf-protection"
+ FSTACK_PREFIX="-fstack-clash-protection"
+
+ for (( j = 0; j < $argc; j++ )); do
+ if [[ ! ${argv[j]} =~ ^${SPECS_FLAG_PREFIX} ]] && \
+ [[ ! ${argv[j]} =~ ^${NO_ERROR_PREFIX} ]] && \
+ [[ ! ${argv[j]} =~ ^${FSTACK_PREFIX} ]] && \
+ [[ ! ${argv[j]} =~ ^${FCF_PROTECTION_PREFIX} ]]; \
+ then
+ CC_OPTIONAL_FLAGS="${CC_OPTIONAL_FLAGS} ${argv[j]}"
+ fi
+ done
+ CC_OPTIONAL_FLAGS=`echo $CC_OPTIONAL_FLAGS | grep ^"-"`
+}
+
+addRequiredFlags()
+{
+ local argc=$#
+ local argv=("$@")
+ CC_OPTIONAL_FLAG_PREFIX="--with-cc-opt="
+ LD_OPTIONAL_FLAG_PREFIX="--with-ld-opt="
+ ADDITIONAL_MODULE_FLAG_PREFIX="--add-module="
+ DYNAMIC_MODULE_FLAG_PREFIX="--add-dynamic-module="
+ BUILD_FLAG_PREFIX="--build="
+ OPENSSL_VERSION_PREFIX="--with-openssl="
+ OPENSSL_OPT_PREFIX="--with-openssl-opt="
+ HPACK_ENC_PREFIX="--with-http_v2_hpack_enc"
+ AUTH_JWT_PREFIX="--with-http_auth_jwt_module"
+ F4F_PREFIX="--with-http_f4f_module"
+ HLS_PREFIX="--with-http_hls_module"
+ SESSION_LOG_PREFIX="--with-http_session_log_module"
+ COMMON_PREFIX="--"
+ WITH_CC_PREFIX="--with-cc"
+
+ for (( i = 1; i < $argc; i++ )); do
+ if [[ "${argv[i]}" =~ ^${COMMON_PREFIX} ]] && \
+ [[ ! ("${argv[i]}" =~ ^${CC_OPTIONAL_FLAG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ^${LD_OPTIONAL_FLAG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${ADDITIONAL_MODULE_FLAG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${OPENSSL_VERSION_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${OPENSSL_OPT_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${DYNAMIC_MODULE_FLAG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${BUILD_FLAG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${AUTH_JWT_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${F4F_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${HLS_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${SESSION_LOG_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${WITH_CC_PREFIX}) ]] && \
+ [[ ! ("${argv[i]}" =~ ${HPACK_ENC_PREFIX}) ]] ; \
+ then
+ debug "Adding configuration flag: ${argv[i]}\n"
+ NUMBER_OF_CONFIGURATION_FLAGS=$((NUMBER_OF_CONFIGURATION_FLAGS+1))
+ CONFIGURATION_FLAGS="${CONFIGURATION_FLAGS} \\\\\n${argv[i]}"
+ fi
+ done
+ COMBINED_CONFIGURATION_FLAGS="${CONFIGURATION_FLAGS}"
+ debug "Successfully added nginx configuration flags"
+}
+
+add_nginx_and_release_versions()
+{
+ echo -e "NGINX_VERSION=\"${NGINX_VERSION}\"" >> ${FILE_NAME}
+ RELEASE_VERSION=`cat /etc/*-release | grep -i "PRETTY_NAME\|Gaia" | cut -d"\"" -f2`
+ echo -e "RELEASE_VERSION=\"${RELEASE_VERSION}\"" >> ${FILE_NAME}
+}
+
+initializeEnviroment
+echo -e "Open-appsec Nginx version extractor\n"
+check_flags_options "$@"
+_main
+
diff --git a/attachments/nginx/ngx_module/ngx_cp_compression.c b/attachments/nginx/ngx_module/ngx_cp_compression.c
index 500e544..35c4d68 100644
--- a/attachments/nginx/ngx_module/ngx_cp_compression.c
+++ b/attachments/nginx/ngx_module/ngx_cp_compression.c
@@ -380,7 +380,11 @@ compression_chain_filter(
}
ngx_memcpy(curr_input_link->buf, output_buffer, sizeof(ngx_buf_t));
- curr_input_link->buf->memory = 1;
+
+ // Empty buffer should not be marked as "in-memory"
+ if (curr_input_link->buf->last - curr_input_link->buf->pos != 0) {
+ curr_input_link->buf->memory = 1;
+ }
}
write_dbg(DBG_LEVEL_TRACE, "Successfully %s chain", should_compress ? "compressed" : "decompressed");
diff --git a/attachments/nginx/ngx_module/ngx_cp_custom_response.c b/attachments/nginx/ngx_module/ngx_cp_custom_response.c
index 28d529f..cc5ba93 100644
--- a/attachments/nginx/ngx_module/ngx_cp_custom_response.c
+++ b/attachments/nginx/ngx_module/ngx_cp_custom_response.c
@@ -333,7 +333,7 @@ ngx_http_cp_finalize_rejected_request(ngx_http_request_t *request)
}
out_chain[0].buf->last_buf = 1;
out_chain[0].next = NULL;
- return ngx_http_output_filter(request, &out_chain[0]);
+ ngx_http_output_filter(request, &out_chain[0]);
}
CUSTOM_RES_OUT:
diff --git a/attachments/nginx/ngx_module/ngx_cp_hook_threads.c b/attachments/nginx/ngx_module/ngx_cp_hook_threads.c
index c8805ee..ce87e7e 100644
--- a/attachments/nginx/ngx_module/ngx_cp_hook_threads.c
+++ b/attachments/nginx/ngx_module/ngx_cp_hook_threads.c
@@ -86,8 +86,8 @@ init_thread_ctx(
void *
ngx_http_cp_registration_thread(void *_ctx)
{
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)_ctx;
- ngx_int_t res = ngx_cp_attachment_init_process();
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
+ ngx_int_t res = ngx_cp_attachment_init_process(ctx->request);
if (res == NGX_ABORT && already_registered) {
already_registered = 0;
disconnect_communication();
@@ -141,7 +141,8 @@ end_req_header_handler(
&session_data_p->verdict,
session_data_p->session_id,
request,
- modifications
+ modifications,
+ REQUEST_END
);
}
@@ -154,7 +155,7 @@ does_contain_body(ngx_http_headers_in_t *headers)
void *
ngx_http_cp_req_header_handler_thread(void *_ctx)
{
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)_ctx;
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
ngx_http_request_t *request = ctx->request;
ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
ngx_int_t send_meta_data_result;
@@ -218,7 +219,7 @@ ngx_http_cp_req_header_handler_thread(void *_ctx)
void *
ngx_http_cp_req_body_filter_thread(void *_ctx)
{
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)_ctx;
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
ngx_http_request_t *request = ctx->request;
ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
ngx_int_t is_last_part;
@@ -248,31 +249,14 @@ ngx_http_cp_req_body_filter_thread(void *_ctx)
}
session_data_p->remaining_messages_to_reply += num_messages_sent;
- num_messages_sent = 0;
- if (is_last_part) {
- // Signals the nano service that the transaction reached the end.
- if (ngx_http_cp_end_transaction_sender(REQUEST_END, session_data_p->session_id, &num_messages_sent) != NGX_OK) {
- write_dbg(
- DBG_LEVEL_WARNING,
- "Failed to send request end data to the nano service. Session ID: %d",
- session_data_p->session_id
- );
- handle_inspection_failure(inspection_failure_weight, fail_mode_verdict, session_data_p);
- if (fail_mode_verdict == NGX_OK) {
- THREAD_CTX_RETURN_NEXT_FILTER();
- }
- THREAD_CTX_RETURN(NGX_ERROR);
- }
- session_data_p->remaining_messages_to_reply++;
- }
-
// Fetch nano services' results.
ctx->res = ngx_http_cp_reply_receiver(
&session_data_p->remaining_messages_to_reply,
&session_data_p->verdict,
session_data_p->session_id,
request,
- &ctx->modifications
+ &ctx->modifications,
+ REQUEST_BODY
);
if (is_last_part) session_data_p->was_request_fully_inspected = 1;
@@ -280,10 +264,49 @@ ngx_http_cp_req_body_filter_thread(void *_ctx)
return NULL;
}
+void *
+ngx_http_cp_req_end_transaction_thread(void *_ctx)
+{
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
+ ngx_http_request_t *request = ctx->request;
+ ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
+ ngx_uint_t num_messages_sent = 0;
+
+ if (ngx_http_cp_end_transaction_sender(REQUEST_END, session_data_p->session_id, &num_messages_sent) != NGX_OK) {
+ write_dbg(
+ DBG_LEVEL_WARNING,
+ "Failed to send request end data to the nano service. Session ID: %d",
+ session_data_p->session_id
+ );
+ handle_inspection_failure(inspection_failure_weight, fail_mode_verdict, session_data_p);
+ if (fail_mode_verdict == NGX_OK) {
+ THREAD_CTX_RETURN_NEXT_FILTER();
+ }
+ THREAD_CTX_RETURN(NGX_ERROR);
+ }
+
+ session_data_p->remaining_messages_to_reply += num_messages_sent;
+
+ if (session_data_p->verdict != TRAFFIC_VERDICT_ACCEPT &&
+ session_data_p->verdict != TRAFFIC_VERDICT_DROP) {
+ // Fetch nano services' results.
+ ctx->res = ngx_http_cp_reply_receiver(
+ &session_data_p->remaining_messages_to_reply,
+ &session_data_p->verdict,
+ session_data_p->session_id,
+ request,
+ &ctx->modifications,
+ REQUEST_END
+ );
+ }
+
+ return NULL;
+}
+
void *
ngx_http_cp_res_header_filter_thread(void *_ctx)
{
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)_ctx;
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
ngx_http_request_t *request = ctx->request;
ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
ngx_int_t set_response_content_encoding_res;
@@ -383,7 +406,8 @@ ngx_http_cp_res_header_filter_thread(void *_ctx)
&session_data_p->verdict,
session_data_p->session_id,
request,
- &ctx->modifications
+ &ctx->modifications,
+ RESPONSE_HEADER
);
return NULL;
@@ -392,7 +416,7 @@ ngx_http_cp_res_header_filter_thread(void *_ctx)
void *
ngx_http_cp_res_body_filter_thread(void *_ctx)
{
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)_ctx;
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
ngx_http_request_t *request = ctx->request;
ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
ngx_int_t send_body_result;
@@ -446,7 +470,50 @@ ngx_http_cp_res_body_filter_thread(void *_ctx)
&session_data_p->verdict,
session_data_p->session_id,
request,
- &ctx->modifications
+ &ctx->modifications,
+ RESPONSE_BODY
+ );
+
+ return NULL;
+}
+
+void *
+ngx_http_cp_hold_verdict_thread(void *_ctx)
+{
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)_ctx;
+
+ ngx_http_request_t *request = ctx->request;
+ ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
+
+ ngx_uint_t num_messages_sent = 0;
+
+ if (ngx_http_cp_wait_sender(session_data_p->session_id, &num_messages_sent) != NGX_OK) {
+ write_dbg(
+ DBG_LEVEL_WARNING,
+ "Failed to send inspect wait request to the nano service. Session ID: %d",
+ session_data_p->session_id
+ );
+ handle_inspection_failure(inspection_failure_weight, fail_mode_hold_verdict, session_data_p);
+ if (fail_mode_hold_verdict == NGX_OK) {
+ THREAD_CTX_RETURN_NEXT_FILTER();
+ }
+ THREAD_CTX_RETURN(NGX_ERROR);
+ }
+ session_data_p->remaining_messages_to_reply += num_messages_sent;
+
+ ctx->res = ngx_http_cp_reply_receiver(
+ &session_data_p->remaining_messages_to_reply,
+ &session_data_p->verdict,
+ session_data_p->session_id,
+ request,
+ &ctx->modifications,
+ HOLD_DATA
+ );
+
+ write_dbg(
+ DBG_LEVEL_TRACE,
+ "Successfully receivied response to wait from the nano service. Session ID: %d",
+ session_data_p->session_id
);
return NULL;
diff --git a/attachments/nginx/ngx_module/ngx_cp_hook_threads.h b/attachments/nginx/ngx_module/ngx_cp_hook_threads.h
index be23896..5619130 100644
--- a/attachments/nginx/ngx_module/ngx_cp_hook_threads.h
+++ b/attachments/nginx/ngx_module/ngx_cp_hook_threads.h
@@ -121,6 +121,19 @@ void * ngx_http_cp_req_header_handler_thread(void *_ctx);
///
void * ngx_http_cp_req_body_filter_thread(void *_ctx);
+///
+/// @brief Sends end request transmission to the attachment's service.
+/// @details Communicates with the attachment service by sending request body to the attachment's service
+/// and modifies _ctx by the received response.
+/// @note _ctx needs to be properly initialized by init_thread_ctx() and ngx_chain_t needs of not NULL.
+/// @param[in, out] _ctx is of type ngx_http_cp_event_thread_ctx_t.
+/// Modifies _ctx res to the following values:
+/// - #NGX_OK
+/// - #NGX_ERROR
+/// @return NULL.
+///
+void * ngx_http_cp_req_end_transaction_thread(void *_ctx);
+
///
/// @brief Sends response headers to the attachment's service.
/// @details Communicates with the attachment service by sending response headers to the attachment's service
@@ -147,6 +160,21 @@ void * ngx_http_cp_res_header_filter_thread(void *_ctx);
///
void * ngx_http_cp_res_body_filter_thread(void *_ctx);
+///
+/// @brief Sends a request to the attachment's service to update the earlier provided "WAIT" verdict.
+/// @details Communicates with the attachment service by sending a HOLD_DATA request to the attachment's service
+/// and modifies _ctx by the received response.
+/// @note _ctx needs to be properly initialized by init_thread_ctx() and
+/// be called after another call returned wait verdict.
+/// @param[in, out] _ctx is of type ngx_http_cp_event_thread_ctx_t.
+/// Modifies _ctx res to the following values:
+/// - #NGX_OK
+/// - #NGX_ERROR
+/// Modifies _ctx session data with an updated verdict.
+/// @return NULL.
+///
+void * ngx_http_cp_hold_verdict_thread(void *_ctx);
+
///
/// @brief Check if transaction contains headers.
/// @param[in] headers ngx_http_headers_in_t struct.
diff --git a/attachments/nginx/ngx_module/ngx_cp_hooks.c b/attachments/nginx/ngx_module/ngx_cp_hooks.c
index 8014d11..3fcb882 100644
--- a/attachments/nginx/ngx_module/ngx_cp_hooks.c
+++ b/attachments/nginx/ngx_module/ngx_cp_hooks.c
@@ -144,6 +144,44 @@ was_transaction_timedout(ngx_http_cp_session_data *ctx)
return 1;
}
+ngx_int_t
+ngx_http_cp_hold_verdict(struct ngx_http_cp_event_thread_ctx_t *ctx)
+{
+ ngx_http_cp_session_data *session_data_p = ctx->session_data_p;
+ for (uint i = 0; i < 3; i++) {
+ sleep(1);
+ int res = ngx_cp_run_in_thread_timeout(
+ ngx_http_cp_hold_verdict_thread,
+ (void *)ctx,
+ waiting_for_verdict_thread_timeout_msec,
+ "ngx_http_cp_hold_verdict_thread"
+ );
+
+ if (!res) {
+ write_dbg(
+ DBG_LEVEL_DEBUG,
+ "ngx_http_cp_hold_verdict_thread failed at attempt number=%d",
+ i
+ );
+ continue;
+ }
+
+ if (session_data_p->verdict != TRAFFIC_VERDICT_WAIT) {
+ // Verdict was updated.
+ write_dbg(
+ DBG_LEVEL_DEBUG,
+ "finished ngx_http_cp_hold_verdict successfully. new verdict=%d",
+ session_data_p->verdict
+ );
+ return 1;
+ }
+ }
+ write_dbg(DBG_LEVEL_TRACE, "Handling Failure with fail %s mode", fail_mode_hold_verdict == NGX_OK ? "open" : "close");
+ handle_inspection_failure(inspection_failure_weight, fail_mode_hold_verdict, session_data_p);
+ session_data_p->verdict = fail_mode_hold_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP;
+ return 0;
+}
+
ngx_http_cp_verdict_e
enforce_sessions_rate()
{
@@ -498,6 +536,12 @@ ngx_http_cp_req_body_filter(ngx_http_request_t *request, ngx_chain_t *request_bo
write_dbg(DBG_LEVEL_DEBUG, "spawn ngx_http_cp_req_body_filter_thread");
// Open threads while unprocessed chain elements still exist, up to num of elements in the chain iterations
for (chain_elem = ctx.chain; chain_elem != NULL && ctx.chain; chain_elem = chain_elem->next) {
+ // Notify if zero-size buf is marked as "memory". This should never happen but if it does we want to know.
+ if (chain_elem->buf && chain_elem->buf->pos &&
+ (chain_elem->buf->last - chain_elem->buf->pos == 0) && chain_elem->buf->memory == 1) {
+ write_dbg(DBG_LEVEL_WARNING,
+ "Warning: encountered request body chain element of size 0 with memory flag enabled");
+ }
clock_gettime(CLOCK_REALTIME, &hook_time_begin);
res = ngx_cp_run_in_thread_timeout(
ngx_http_cp_req_body_filter_thread,
@@ -529,8 +573,45 @@ ngx_http_cp_req_body_filter(ngx_http_request_t *request, ngx_chain_t *request_bo
ctx.res
);
- calcProcessingTime(session_data_p, &hook_time_begin, 1);
+ if (session_data_p->verdict == TRAFFIC_VERDICT_WAIT) {
+ res = ngx_http_cp_hold_verdict(&ctx);
+ if (!res) {
+ session_data_p->verdict = fail_mode_hold_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP;
+ updateMetricField(HOLD_THREAD_TIMEOUT, 1);
+ return fail_mode_verdict == NGX_OK ? ngx_http_next_request_body_filter(request, request_body_chain) : NGX_ERROR;
+ }
+ }
+ if (session_data_p->was_request_fully_inspected) {
+ res = ngx_cp_run_in_thread_timeout(
+ ngx_http_cp_req_end_transaction_thread,
+ (void *)&ctx,
+ req_body_thread_timeout_msec,
+ "ngx_http_cp_req_end_transaction_thread"
+ );
+ if (!res) {
+ // failed to execute thread task, or it timed out
+ session_data_p->verdict = fail_mode_verdict == NGX_OK ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP;
+ write_dbg(
+ DBG_LEVEL_DEBUG,
+ "req_end_transaction thread failed, returning default fail mode verdict. Session id: %d, verdict: %s",
+ session_data_p->session_id,
+ session_data_p->verdict == TRAFFIC_VERDICT_ACCEPT ? "accept" : "drop"
+ );
+ updateMetricField(REQ_BODY_THREAD_TIMEOUT, 1);
+ return fail_mode_verdict == NGX_OK ? ngx_http_next_request_body_filter(request, request_body_chain) : NGX_ERROR;
+ }
+
+ write_dbg(
+ DBG_LEVEL_DEBUG,
+ "finished ngx_http_cp_req_end_transaction_thread successfully. return=%d next_filter=%d res=%d",
+ ctx.should_return,
+ ctx.should_return_next_filter,
+ ctx.res
+ );
+ }
+
+ calcProcessingTime(session_data_p, &hook_time_begin, 1);
if (ctx.should_return_next_filter) {
return ngx_http_next_request_body_filter(request, request_body_chain);
}
@@ -787,7 +868,7 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain
if (session_data_p->verdict == TRAFFIC_VERDICT_DROP) request->keepalive = 0;
return ngx_http_next_response_body_filter(request, body_chain);
}
-
+
session_data_p->response_data.num_body_chunk++;
if (body_chain == NULL) {
@@ -890,6 +971,12 @@ ngx_http_cp_res_body_filter(ngx_http_request_t *request, ngx_chain_t *body_chain
write_dbg(DBG_LEVEL_DEBUG, "spawn ngx_http_cp_res_body_filter_thread");
// Open threads while unprocessed chain elements still exist, up to num of elements in the chain iterations
for (chain_elem = ctx.chain; chain_elem != NULL && ctx.chain; chain_elem = chain_elem->next) {
+ // Notify if zero-size buf is marked as "memory". This should never happen but if it does we want to know.
+ if (chain_elem->buf && chain_elem->buf->pos &&
+ (chain_elem->buf->last - chain_elem->buf->pos == 0) && chain_elem->buf->memory == 1) {
+ write_dbg(DBG_LEVEL_WARNING,
+ "Warning: encountered response body chain element of size 0 with memory flag enabled");
+ }
clock_gettime(CLOCK_REALTIME, &hook_time_begin);
if (!ngx_cp_run_in_thread_timeout(
ngx_http_cp_res_body_filter_thread,
diff --git a/attachments/nginx/ngx_module/ngx_cp_hooks.h b/attachments/nginx/ngx_module/ngx_cp_hooks.h
index 6ed097b..a8c5c00 100644
--- a/attachments/nginx/ngx_module/ngx_cp_hooks.h
+++ b/attachments/nginx/ngx_module/ngx_cp_hooks.h
@@ -25,6 +25,7 @@
#include "ngx_cp_http_parser.h"
#include "nginx_attachment_common.h"
+#include "ngx_cp_hook_threads.h"
static const int registration_failure_weight = 2; ///< Registration failure weight.
static const int inspection_failure_weight = 1; ///< Inspection failure weight.
@@ -96,6 +97,16 @@ ngx_int_t ngx_http_cp_res_header_filter(ngx_http_request_t *request);
///
ngx_int_t ngx_http_cp_req_header_handler(ngx_http_request_t *request);
+///
+/// @brief Sends a request to the nano service to update the verdict.
+/// @note Should be called after the nano service provided the verdict TRAFFIC_VERDICT_WAIT to get the updated verdict.
+/// @param[in, out] request Event thread context to be updated.
+/// @returns ngx_int_t
+/// - #1 if request was properly communicated with the nano service and provided an updated response.
+/// - #0 otherwise.
+///
+ngx_int_t ngx_http_cp_hold_verdict(struct ngx_http_cp_event_thread_ctx_t *ctx);
+
///
/// @brief Checks if transaction was timed out.
/// @param[in, out] ctx
diff --git a/attachments/nginx/ngx_module/ngx_cp_initializer.h b/attachments/nginx/ngx_module/ngx_cp_initializer.h
index 2c00b12..ed30a7e 100644
--- a/attachments/nginx/ngx_module/ngx_cp_initializer.h
+++ b/attachments/nginx/ngx_module/ngx_cp_initializer.h
@@ -32,7 +32,7 @@ typedef enum ngx_cp_comm_direction {
/// - #NGX_ERROR
/// - #NGX_ABORT
///
-ngx_int_t ngx_cp_attachment_init_process();
+ngx_int_t ngx_cp_attachment_init_process(ngx_http_request_t *);
///
/// @brief Preforms send\receive information to\from the service via a socket.
diff --git a/attachments/nginx/ngx_module/ngx_cp_io.c b/attachments/nginx/ngx_module/ngx_cp_io.c
index 5110807..c997164 100644
--- a/attachments/nginx/ngx_module/ngx_cp_io.c
+++ b/attachments/nginx/ngx_module/ngx_cp_io.c
@@ -81,6 +81,7 @@ ngx_http_cp_signal_to_service(uint32_t cur_session_id)
///
/// @brief Signals and recieve signal to/from nano service about new session to inspect.
/// @param[in] cur_session_id Session's Id.
+/// @param[in] chunk_type Chunk type that the attachment is waiting for a response from nano service.
/// @returns ngx_int_t
/// - #NGX_OK
/// - #NGX_ERROR
@@ -88,7 +89,7 @@ ngx_http_cp_signal_to_service(uint32_t cur_session_id)
/// - #NGX_AGAIN
///
static ngx_int_t
-ngx_http_cp_wait_for_service(uint32_t cur_session_id)
+ngx_http_cp_wait_for_service(uint32_t cur_session_id, ngx_http_chunk_type_e chunk_type)
{
static int dbg_count = 0;
static clock_t clock_start = (clock_t) 0;
@@ -97,6 +98,7 @@ ngx_http_cp_wait_for_service(uint32_t cur_session_id)
uint32_t reply_from_service;
ngx_int_t retry;
int is_fail_open_disabled = (inspection_mode != NON_BLOCKING_THREAD);
+ ngx_uint_t timeout = chunk_type == HOLD_DATA ? fail_open_hold_timeout : fail_open_timeout;
res = ngx_http_cp_signal_to_service(cur_session_id);
if (res != NGX_OK) return res;
@@ -111,7 +113,7 @@ ngx_http_cp_wait_for_service(uint32_t cur_session_id)
s_poll.fd = comm_socket;
s_poll.events = POLLIN;
s_poll.revents = 0;
- res = poll(&s_poll, 1, is_fail_open_disabled ? 150 : fail_open_timeout);
+ res = poll(&s_poll, 1, is_fail_open_disabled ? 150 : timeout);
if (res < 0) {
// Polling from the nano service has failed.
@@ -192,7 +194,9 @@ ngx_http_cp_send_data_to_service(
const uint16_t *fragments_sizes,
uint8_t num_of_data_elem,
uint32_t cur_session_id,
- int *was_waiting)
+ int *was_waiting,
+ ngx_http_chunk_type_e chunk_type
+)
{
ngx_int_t max_retries;
ngx_int_t res = NGX_OK;
@@ -209,7 +213,7 @@ ngx_http_cp_send_data_to_service(
*was_waiting = 1;
}
- res = ngx_http_cp_wait_for_service(cur_session_id);
+ res = ngx_http_cp_wait_for_service(cur_session_id, chunk_type);
if (res != NGX_OK && res != NGX_AGAIN) return res;
}
@@ -428,7 +432,9 @@ ngx_http_cp_reply_receiver(
ngx_http_cp_verdict_e *verdict,
uint32_t cur_session_id,
ngx_http_request_t *request,
- ngx_http_cp_modification_list **modification_list)
+ ngx_http_cp_modification_list **modification_list,
+ ngx_http_chunk_type_e chunk_type
+)
{
ngx_http_cp_reply_from_service_t *reply_p;
ngx_http_cp_modification_list *new_modification = NULL;
@@ -446,7 +452,7 @@ ngx_http_cp_reply_receiver(
}
do {
- res = ngx_http_cp_wait_for_service(cur_session_id);
+ res = ngx_http_cp_wait_for_service(cur_session_id, chunk_type);
} while (res == NGX_AGAIN);
if (res != NGX_OK) return NGX_ERROR;
@@ -554,10 +560,19 @@ ngx_http_cp_reply_receiver(
break;
}
- case TRAFFIC_VERDICT_INSPECT:
+ case TRAFFIC_VERDICT_INSPECT: {
// After an irrelevant verdict, ignore the verdict and continue to the next response.
+ write_dbg(DBG_LEVEL_TRACE, "Verdict inspect received from the nano service");
updateMetricField(INSPECT_VERDICTS_COUNT, 1);
break;
+ }
+
+ case TRAFFIC_VERDICT_WAIT: {
+ // After a wait verdict, query the nano agent again to get an updated verdict.
+ write_dbg(DBG_LEVEL_DEBUG, "Verdict wait received from the nano service");
+ updateMetricField(HOLD_VERDICTS_COUNT, 1);
+ break;
+ }
}
free_data_from_service();
@@ -718,7 +733,7 @@ ngx_http_cp_meta_data_sender(ngx_http_request_t *request, uint32_t cur_request_i
set_fragment_elem(fragments, fragments_sizes, &client_port, sizeof(client_port), CLIENT_PORT + 2);
// Sends all the data to the nano service.
- res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, META_DATA_COUNT + 2, cur_request_id, NULL);
+ res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, META_DATA_COUNT + 2, cur_request_id, NULL, fail_open_timeout);
if (res != NGX_OK) {
// Failed to send the metadata to nano service.
if (res == NGX_ERROR && failure_count++ == 5) {
@@ -764,7 +779,7 @@ ngx_http_cp_end_transaction_sender(
set_fragments_identifiers(fragments, fragments_sizes, (uint16_t *)&end_transaction_type, &cur_request_id);
- res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, end_transaction_num_fragments, cur_request_id, NULL);
+ res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, end_transaction_num_fragments, cur_request_id, NULL, fail_open_timeout);
if (res != NGX_OK) {
return NGX_ERROR;
}
@@ -773,6 +788,30 @@ ngx_http_cp_end_transaction_sender(
return NGX_OK;
}
+ngx_int_t
+ngx_http_cp_wait_sender(uint32_t cur_request_id, ngx_uint_t *num_messages_sent)
+{
+ static const ngx_uint_t end_transaction_num_fragments = 2;
+
+ char *fragments[end_transaction_num_fragments];
+ uint16_t fragments_sizes[end_transaction_num_fragments];
+ ngx_http_chunk_type_e transaction_type = HOLD_DATA;
+ ngx_int_t res;
+
+ set_fragments_identifiers(fragments, fragments_sizes, (uint16_t *)&transaction_type, &cur_request_id);
+
+ write_dbg(DBG_LEVEL_TRACE, "Sending wait event flag for inspection");
+
+ res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, end_transaction_num_fragments, cur_request_id, NULL, fail_open_timeout);
+ if (res != NGX_OK) {
+ return NGX_ERROR;
+ }
+
+ write_dbg(DBG_LEVEL_TRACE, "Successfully sent wait event");
+ *num_messages_sent = 1;
+ return NGX_OK;
+}
+
ngx_int_t
ngx_http_cp_res_code_sender(uint16_t response_code, uint32_t cur_req_id, ngx_uint_t *num_messages_sent)
{
@@ -788,7 +827,7 @@ ngx_http_cp_res_code_sender(uint16_t response_code, uint32_t cur_req_id, ngx_uin
set_fragments_identifiers(fragments, fragments_sizes, &chunck_type, &cur_req_id);
set_fragment_elem(fragments, fragments_sizes, &response_code, sizeof(uint16_t), 2);
- if (ngx_http_cp_send_data_to_service(fragments, fragments_sizes, res_code_num_fragments, cur_req_id, NULL) != NGX_OK) {
+ if (ngx_http_cp_send_data_to_service(fragments, fragments_sizes, res_code_num_fragments, cur_req_id, NULL, fail_open_hold_timeout) != NGX_OK) {
return NGX_ERROR;
}
@@ -812,7 +851,7 @@ ngx_http_cp_content_length_sender(uint64_t content_length_n, uint32_t cur_req_id
set_fragments_identifiers(fragments, fragments_sizes, &chunck_type, &cur_req_id);
set_fragment_elem(fragments, fragments_sizes, &content_length_val, sizeof(content_length_val), 2);
- if (ngx_http_cp_send_data_to_service(fragments, fragments_sizes, content_length_num_fragments, cur_req_id, NULL) != NGX_OK) {
+ if (ngx_http_cp_send_data_to_service(fragments, fragments_sizes, content_length_num_fragments, cur_req_id, NULL, fail_open_timeout) != NGX_OK) {
return NGX_ERROR;
}
@@ -865,7 +904,7 @@ send_header_bulk(
set_fragment_elem(data, data_sizes, &is_last_part, sizeof(is_last_part), 2);
set_fragment_elem(data, data_sizes, &bulk_part_index, sizeof(bulk_part_index), 3);
- res = ngx_http_cp_send_data_to_service(data, data_sizes, HEADER_DATA_COUNT * num_headers + 4, cur_request_id, NULL);
+ res = ngx_http_cp_send_data_to_service(data, data_sizes, HEADER_DATA_COUNT * num_headers + 4, cur_request_id, NULL, fail_open_timeout);
if (res != NGX_OK) {
write_dbg(DBG_LEVEL_TRACE, "Failed to send bulk of %iu headers", num_headers);
return NGX_ERROR;
@@ -1048,7 +1087,7 @@ ngx_http_cp_body_sender(
}
// Sending the data to the nano service.
res = ngx_http_cp_send_data_to_service(fragments, fragments_sizes, num_body_chunk_fragments, session_data->session_id,
- &was_waiting);
+ &was_waiting, fail_open_timeout);
if (res != NGX_OK) {
// Failed to send the fragments to the nano service.
@@ -1091,7 +1130,7 @@ ngx_http_cp_metric_data_sender()
fragments = (char *)&data_to_send;
fragments_sizes = sizeof(ngx_http_cp_metric_data_t);
- res = ngx_http_cp_send_data_to_service(&fragments, &fragments_sizes, 1, 0, NULL);
+ res = ngx_http_cp_send_data_to_service(&fragments, &fragments_sizes, 1, 0, NULL, fail_open_timeout);
reset_metric_data();
return res;
}
diff --git a/attachments/nginx/ngx_module/ngx_cp_io.h b/attachments/nginx/ngx_module/ngx_cp_io.h
index 8046c24..d662bfe 100644
--- a/attachments/nginx/ngx_module/ngx_cp_io.h
+++ b/attachments/nginx/ngx_module/ngx_cp_io.h
@@ -47,6 +47,7 @@ extern int comm_socket; ///< Communication socket.
/// @param[in] cur_session_id Session's Id.
/// @param[in, out] request NGINX request.
/// @param[in] modification_list
+/// @param[in] chunk_type Chunk type that the attachment is waiting for a response from nano service.
/// @returns ngx_int_t
/// - #NGX_OK
/// - #NGX_HTTP_FORBIDDEN
@@ -58,7 +59,8 @@ ngx_http_cp_reply_receiver(
ngx_http_cp_verdict_e *verdict,
uint32_t cur_session_id,
ngx_http_request_t *request,
- ngx_http_cp_modification_list **modification_list
+ ngx_http_cp_modification_list **modification_list,
+ ngx_http_chunk_type_e chunk_type
);
///
@@ -170,6 +172,17 @@ ngx_http_cp_body_sender(
ngx_chain_t **next_elem_to_inspect
);
+///
+/// @brief Sends HOLD_DATA request to the nano service.
+/// @details HOLD_DATA request is a request that asks the nano service to provide with an updated verdict.
+/// @param[in] cur_request_id Request session's Id.
+/// @param[in, out] num_messages_sent Number of messages sent will be saved onto this parameter.
+/// - #NGX_OK
+/// - #NGX_ERROR
+///
+ngx_int_t
+ngx_http_cp_wait_sender(uint32_t cur_request_id, ngx_uint_t *num_messages_sent);
+
///
/// @brief Checks if reconf is needed and reconfigs if necessary.
/// @returns ngx_int_t
diff --git a/attachments/nginx/ngx_module/ngx_cp_thread.c b/attachments/nginx/ngx_module/ngx_cp_thread.c
index ec3d691..11c4ead 100644
--- a/attachments/nginx/ngx_module/ngx_cp_thread.c
+++ b/attachments/nginx/ngx_module/ngx_cp_thread.c
@@ -51,7 +51,7 @@ ngx_cp_run_in_thread_timeout(CpThreadRoutine thread_func, void *arg, int timeout
void *res = NULL;
pthread_t thread;
struct timespec ts;
- struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t*)arg;
+ struct ngx_http_cp_event_thread_ctx_t *ctx = (struct ngx_http_cp_event_thread_ctx_t *)arg;
if (inspection_mode == NO_THREAD) return ngx_cp_run_without_thread_timeout(thread_func, arg, func_name);
diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.c b/attachments/nginx/ngx_module/ngx_cp_utils.c
index 81cbfc2..72f989a 100644
--- a/attachments/nginx/ngx_module/ngx_cp_utils.c
+++ b/attachments/nginx/ngx_module/ngx_cp_utils.c
@@ -81,9 +81,11 @@ ngx_http_cp_sessions_per_minute_limit sessions_per_minute_limit_info = {
ngx_uint_t current_config_version = 0;
ngx_int_t fail_mode_verdict = NGX_OK; ///< Fail open verdict incase of a timeout.
+ngx_int_t fail_mode_hold_verdict = NGX_OK; ///< Fail open verdict incase of a timeout when waiting for wait verdict.
ngx_int_t dbg_is_needed = 0; ///< Debug flag.
ngx_int_t num_of_connection_attempts = 0; ///< Maximum number of attempted connections.
ngx_uint_t fail_open_timeout = 50; ///< Fail open timeout in milliseconds.
+ngx_uint_t fail_open_hold_timeout = 150; ///< Fail open wait timeout in milliseconds.
ngx_http_cp_verdict_e sessions_per_minute_limit_verdict = TRAFFIC_VERDICT_ACCEPT;
ngx_uint_t max_sessions_per_minute = 0; ///< Masimum session per minute.
ngx_uint_t req_max_proccessing_ms_time = 3000; ///< Total Request processing timeout in milliseconds.
@@ -93,6 +95,7 @@ ngx_uint_t req_header_thread_timeout_msec = 100; ///< Request header processing
ngx_uint_t req_body_thread_timeout_msec = 150; ///< Request body processing timeout in milliseconds.
ngx_uint_t res_header_thread_timeout_msec = 100; ///< Response header processing timeout in milliseconds.
ngx_uint_t res_body_thread_timeout_msec = 150; ///< Response body processing timeout in milliseconds.
+ngx_uint_t waiting_for_verdict_thread_timeout_msec = 150; ///< Wait thread processing timeout in milliseconds.
ngx_http_inspection_mode_e inspection_mode = NON_BLOCKING_THREAD; ///< Default inspection mode.
ngx_uint_t num_of_nginx_ipc_elements = 200; ///< Number of NGINX IPC elements.
ngx_msec_t keep_alive_interval_msec = DEFAULT_KEEP_ALIVE_INTERVAL_MSEC;
@@ -892,6 +895,10 @@ init_general_config(const char *conf_path)
fail_mode_verdict = isFailOpenMode() == 1 ? NGX_OK : NGX_ERROR;
fail_open_timeout = getFailOpenTimeout();
+ // Setting fail wait open/close
+ fail_mode_hold_verdict = isFailOpenHoldMode() == 1 ? NGX_OK : NGX_ERROR;
+ fail_open_hold_timeout = getFailOpenHoldTimeout();
+
// Setting attachment's variables.
sessions_per_minute_limit_verdict = isFailOpenOnSessionLimit() ? TRAFFIC_VERDICT_ACCEPT : TRAFFIC_VERDICT_DROP;
max_sessions_per_minute = getMaxSessionsPerMinute();
@@ -903,6 +910,7 @@ init_general_config(const char *conf_path)
req_body_thread_timeout_msec = getReqBodyThreadTimeout();
res_header_thread_timeout_msec = getResHeaderThreadTimeout();
res_body_thread_timeout_msec = getResBodyThreadTimeout();
+ waiting_for_verdict_thread_timeout_msec = getWaitingForVerdictThreadTimeout();
num_of_nginx_ipc_elements = getNumOfNginxIpcElements();
keep_alive_interval_msec = (ngx_msec_t) getKeepAliveIntervalMsec();
@@ -917,6 +925,8 @@ init_general_config(const char *conf_path)
"debug level: %d, "
"failure mode: %s, "
"fail mode timeout: %u msec, "
+ "failure wait mode: %s, "
+ "fail mode wait timeout: %u msec, "
"sessions per minute limit verdict: %s, "
"max sessions per minute: %u, "
"req max processing time: %u msec, "
@@ -926,6 +936,7 @@ init_general_config(const char *conf_path)
"req body thread timeout: %u msec, "
"res header thread timeout: %u msec, "
"res body thread timeout: %u msec, "
+ "wait thread timeout: %u msec, "
"static resources path: %s, "
"num of nginx ipc elements: %u, "
"keep alive interval msec: %u msec",
@@ -933,6 +944,8 @@ init_general_config(const char *conf_path)
new_dbg_level,
(fail_mode_verdict == NGX_OK ? "fail-open" : "fail-close"),
fail_open_timeout,
+ (fail_mode_hold_verdict == NGX_OK ? "fail-open" : "fail-close"),
+ fail_open_hold_timeout,
sessions_per_minute_limit_verdict == TRAFFIC_VERDICT_ACCEPT ? "Accpet" : "Drop",
max_sessions_per_minute,
req_max_proccessing_ms_time,
@@ -942,6 +955,7 @@ init_general_config(const char *conf_path)
req_body_thread_timeout_msec,
res_header_thread_timeout_msec,
res_body_thread_timeout_msec,
+ waiting_for_verdict_thread_timeout_msec,
getStaticResourcesPath(),
num_of_nginx_ipc_elements,
keep_alive_interval_msec
diff --git a/attachments/nginx/ngx_module/ngx_cp_utils.h b/attachments/nginx/ngx_module/ngx_cp_utils.h
index 50882e4..294712d 100644
--- a/attachments/nginx/ngx_module/ngx_cp_utils.h
+++ b/attachments/nginx/ngx_module/ngx_cp_utils.h
@@ -46,10 +46,12 @@
}
extern ngx_int_t fail_mode_verdict;
+extern ngx_int_t fail_mode_hold_verdict;
extern ngx_int_t dbg_is_needed;
extern ngx_int_t num_of_connection_attempts;
extern ngx_uint_t content_length_would_change;
extern ngx_uint_t fail_open_timeout;
+extern ngx_uint_t fail_open_hold_timeout;
extern ngx_uint_t req_max_proccessing_ms_time;
extern ngx_uint_t res_max_proccessing_ms_time;
extern ngx_uint_t registration_thread_timeout_msec;
@@ -57,6 +59,7 @@ extern ngx_uint_t req_header_thread_timeout_msec;
extern ngx_uint_t req_body_thread_timeout_msec;
extern ngx_uint_t res_header_thread_timeout_msec;
extern ngx_uint_t res_body_thread_timeout_msec;
+extern ngx_uint_t waiting_for_verdict_thread_timeout_msec;
extern ngx_http_inspection_mode_e inspection_mode;
extern ngx_uint_t num_of_nginx_ipc_elements;
diff --git a/attachments/nginx/ngx_module/ngx_modules.c b/attachments/nginx/ngx_module/ngx_modules.c
new file mode 100755
index 0000000..82eb10b
--- /dev/null
+++ b/attachments/nginx/ngx_module/ngx_modules.c
@@ -0,0 +1,37 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/// @file ngx_modules.c
+/// \brief This file is generated from configuration and hold the data structures connecting the plugin to the server
+
+#include
+#include
+
+extern ngx_module_t ngx_http_cp_attachment_module;
+
+ngx_module_t *ngx_modules[] = {
+ &ngx_http_cp_attachment_module,
+ NULL
+};
+
+char *ngx_module_names[] = {
+ "ngx_http_cp_attachment_module",
+ NULL
+};
+
+char *ngx_module_order[] = {
+ "ngx_http_cp_attachment_module",
+ "ngx_http_copy_filter_module",
+ NULL
+};
diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt
index 7482955..3c75a10 100644
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -1,2 +1,3 @@
add_subdirectory(shmem_ipc)
add_subdirectory(compression)
+add_subdirectory(attachments)
diff --git a/core/attachments/CMakeLists.txt b/core/attachments/CMakeLists.txt
new file mode 100644
index 0000000..0b0657b
--- /dev/null
+++ b/core/attachments/CMakeLists.txt
@@ -0,0 +1 @@
+add_subdirectory(http_configuration)
diff --git a/core/attachments/http_configuration/CMakeLists.txt b/core/attachments/http_configuration/CMakeLists.txt
new file mode 100644
index 0000000..424fa41
--- /dev/null
+++ b/core/attachments/http_configuration/CMakeLists.txt
@@ -0,0 +1 @@
+add_library(http_configuration http_configuration.cc)
diff --git a/core/attachments/http_configuration/http_configuration.cc b/core/attachments/http_configuration/http_configuration.cc
new file mode 100644
index 0000000..ecb001b
--- /dev/null
+++ b/core/attachments/http_configuration/http_configuration.cc
@@ -0,0 +1,220 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "http_configuration.h"
+
+#include
+
+#include "cereal/types/vector.hpp"
+
+#define DEFAULT_KEEP_ALIVE_INTERVAL_MSEC 30000
+
+using namespace std;
+
+void
+DebugConfig::save(cereal::JSONOutputArchive &archive) const
+{
+ archive(
+ cereal::make_nvp("clientIp", client),
+ cereal::make_nvp("listeningIp", server),
+ cereal::make_nvp("uriPrefix", uri),
+ cereal::make_nvp("hostName", host),
+ cereal::make_nvp("httpMethod", method),
+ cereal::make_nvp("listeningPort", port)
+ );
+}
+
+void
+DebugConfig::load(cereal::JSONInputArchive &archive)
+{
+ try {
+ archive(
+ cereal::make_nvp("clientIp", client),
+ cereal::make_nvp("listeningIp", server),
+ cereal::make_nvp("uriPrefix", uri),
+ cereal::make_nvp("hostName", host),
+ cereal::make_nvp("httpMethod", method),
+ cereal::make_nvp("listeningPort", port)
+ );
+ } catch (const cereal::Exception &) {
+ client = "";
+ server = "";
+ uri = "";
+ host = "";
+ method = "";
+ port = 0;
+ }
+}
+
+bool
+DebugConfig::operator==(const DebugConfig &another) const
+{
+ return
+ client == another.client &&
+ server == another.server &&
+ port == another.port &&
+ method == another.method &&
+ host == another.host &&
+ uri == another.uri;
+}
+
+int
+HttpAttachmentConfiguration::init(const string &conf_file)
+{
+ try {
+ ifstream file(conf_file);
+ cereal::JSONInputArchive ar(file);
+ load(ar);
+ return 1;
+ } catch (exception &e) {
+ return 0;
+ }
+}
+
+void
+HttpAttachmentConfiguration::save(cereal::JSONOutputArchive &archive) const
+{
+ archive(
+ cereal::make_nvp("context_values", dbg),
+ cereal::make_nvp("ip_ranges", exclude_sources),
+ cereal::make_nvp("dbg_level", getNumericalValue("dbg_level")),
+ cereal::make_nvp("static_resources_path", getStringValue("static_resources_path")),
+ cereal::make_nvp("is_fail_open_mode_enabled", getNumericalValue("is_fail_open_mode_enabled")),
+ cereal::make_nvp("fail_open_timeout", getNumericalValue("fail_open_timeout")),
+ cereal::make_nvp("is_fail_open_mode_hold_enabled", getNumericalValue("is_fail_open_mode_hold_enabled")),
+ cereal::make_nvp("fail_open_hold_timeout", getNumericalValue("fail_open_hold_timeout")),
+ cereal::make_nvp("sessions_per_minute_limit_verdict", getStringValue("sessions_per_minute_limit_verdict")),
+ cereal::make_nvp("max_sessions_per_minute", getNumericalValue("max_sessions_per_minute")),
+ cereal::make_nvp("res_proccessing_timeout_msec", getNumericalValue("res_proccessing_timeout_msec")),
+ cereal::make_nvp("req_proccessing_timeout_msec", getNumericalValue("req_proccessing_timeout_msec")),
+ cereal::make_nvp("registration_thread_timeout_msec", getNumericalValue("registration_thread_timeout_msec")),
+ cereal::make_nvp("req_header_thread_timeout_msec", getNumericalValue("req_header_thread_timeout_msec")),
+ cereal::make_nvp("req_body_thread_timeout_msec", getNumericalValue("req_body_thread_timeout_msec")),
+ cereal::make_nvp("res_header_thread_timeout_msec", getNumericalValue("res_header_thread_timeout_msec")),
+ cereal::make_nvp("res_body_thread_timeout_msec", getNumericalValue("res_body_thread_timeout_msec")),
+ cereal::make_nvp(
+ "waiting_for_verdict_thread_timeout_msec",
+ getNumericalValue("waiting_for_verdict_thread_timeout_msec")
+ ),
+ cereal::make_nvp("nginx_inspection_mode", getNumericalValue("inspection_mode")),
+ cereal::make_nvp("num_of_nginx_ipc_elements", getNumericalValue("num_of_nginx_ipc_elements")),
+ cereal::make_nvp("keep_alive_interval_msec", getNumericalValue("keep_alive_interval_msec"))
+ );
+}
+
+void
+HttpAttachmentConfiguration::load(cereal::JSONInputArchive &archive)
+{
+ try {
+ archive(cereal::make_nvp("context_values", dbg));
+ } catch (const cereal::Exception &) {
+ dbg = DebugConfig();
+ }
+
+ try {
+ archive(cereal::make_nvp("ip_ranges", exclude_sources));
+ } catch (const cereal::Exception &) {
+ exclude_sources = {};
+ }
+
+ try {
+ string str;
+ archive(cereal::make_nvp("static_resources_path", str));
+ string_values["static_resources_path"] = str;
+ } catch (const cereal::Exception &) {
+ string_values.erase("static_resources_path");
+ }
+
+ try {
+ string str;
+ archive(cereal::make_nvp("sessions_per_minute_limit_verdict", str));
+ string_values["sessions_per_minute_limit_verdict"] = str;
+ } catch (const cereal::Exception &) {
+ string_values.erase("sessions_per_minute_limit_verdict");
+ }
+
+ loadNumericalValue(archive, "dbg_level", 0);
+ loadNumericalValue(archive, "is_fail_open_mode_enabled", 0);
+ loadNumericalValue(archive, "fail_open_timeout", 50);
+ loadNumericalValue(archive, "is_fail_open_mode_hold_enabled", 0);
+ loadNumericalValue(archive, "fail_open_hold_timeout", 200);
+ loadNumericalValue(archive, "sessions_per_minute_limit_verdict", 0);
+ loadNumericalValue(archive, "max_sessions_per_minute", 0);
+ loadNumericalValue(archive, "res_proccessing_timeout_msec", 3000);
+ loadNumericalValue(archive, "req_proccessing_timeout_msec", 3000);
+ loadNumericalValue(archive, "registration_thread_timeout_msec", 100);
+ loadNumericalValue(archive, "req_header_thread_timeout_msec", 100);
+ loadNumericalValue(archive, "req_body_thread_timeout_msec", 150);
+ loadNumericalValue(archive, "res_header_thread_timeout_msec", 100);
+ loadNumericalValue(archive, "res_body_thread_timeout_msec", 150);
+ loadNumericalValue(archive, "waiting_for_verdict_thread_timeout_msec", 150);
+ loadNumericalValue(archive, "nginx_inspection_mode", 0);
+ loadNumericalValue(archive, "num_of_nginx_ipc_elements", 200);
+ loadNumericalValue(archive, "keep_alive_interval_msec", DEFAULT_KEEP_ALIVE_INTERVAL_MSEC);
+
+ try {
+ string user_check_logo_path;
+ archive(cereal::make_nvp("user_check_logo_path", user_check_logo_path));
+ ifstream logo_file(user_check_logo_path);
+ if (!logo_file.is_open()) {
+ string_values.erase("user_check_logo");
+ } else {
+ string_values["user_check_logo"] = string(
+ istreambuf_iterator(logo_file),
+ istreambuf_iterator()
+ );
+ }
+ } catch (const cereal::Exception &) {
+ string_values.erase("user_check_logo");
+ }
+}
+
+bool
+HttpAttachmentConfiguration::operator==(const HttpAttachmentConfiguration &other) const
+{
+ return
+ dbg == other.dbg &&
+ numerical_values == other.numerical_values &&
+ string_values == other.string_values &&
+ exclude_sources == other.exclude_sources;
+}
+
+unsigned int
+HttpAttachmentConfiguration::getNumericalValue(const string &key) const
+{
+ auto elem = numerical_values.find(key);
+ return elem != numerical_values.end() ? elem->second : 0;
+}
+
+const string &
+HttpAttachmentConfiguration::getStringValue(const string &key) const
+{
+ auto elem = string_values.find(key);
+ return elem != string_values.end() ? elem->second : empty;
+}
+
+void
+HttpAttachmentConfiguration::loadNumericalValue(
+ cereal::JSONInputArchive &ar,
+ const string &name,
+ unsigned int default_value
+)
+{
+ try {
+ unsigned int value;
+ ar(cereal::make_nvp(name, value));
+ numerical_values[name] = value;
+ } catch (const cereal::Exception &) {
+ numerical_values[name] = default_value;
+ }
+}
diff --git a/core/compression/compression_utils.cc b/core/compression/compression_utils.cc
index 85414ac..2b79d45 100644
--- a/core/compression/compression_utils.cc
+++ b/core/compression/compression_utils.cc
@@ -105,6 +105,7 @@ static const int zlib_no_flush = Z_NO_FLUSH;
struct CompressionStream
{
CompressionStream() { bzero(&stream, sizeof(z_stream)); }
+ ~CompressionStream() { fini(); }
tuple, bool>
decompress(const unsigned char *data, uint32_t size)
diff --git a/core/include/attachments/http_configuration.h b/core/include/attachments/http_configuration.h
index 8c1bd77..20573cb 100644
--- a/core/include/attachments/http_configuration.h
+++ b/core/include/attachments/http_configuration.h
@@ -29,7 +29,7 @@ struct DebugConfig
std::string client;
std::string server;
- uint port = 0;
+ unsigned int port = 0;
std::string method;
std::string host;
std::string uri;
@@ -45,21 +45,21 @@ public:
bool operator==(const HttpAttachmentConfiguration &other) const;
- uint getNumericalValue(const std::string &key) const;
+ unsigned int getNumericalValue(const std::string &key) const;
const std::string & getStringValue(const std::string &key) const;
const std::vector & getExcludeSources() const { return exclude_sources; }
const DebugConfig & getDebugContext() const { return dbg; }
- void setNumericalValue(const std::string &key, uint value) { numerical_values[key] = value; }
+ void setNumericalValue(const std::string &key, unsigned int value) { numerical_values[key] = value; }
void setStringValue(const std::string &key, const std::string &value) { string_values[key] = value; }
void setExcludeSources(const std::vector &new_sources) { exclude_sources = new_sources; }
void setDebugContext(const DebugConfig &_dbg) { dbg = _dbg; }
private:
- void loadNumericalValue(cereal::JSONInputArchive &archive, const std::string &name, uint default_value);
+ void loadNumericalValue(cereal::JSONInputArchive &archive, const std::string &name, unsigned int default_value);
DebugConfig dbg;
- std::map numerical_values;
+ std::map numerical_values;
std::map string_values;
std::vector exclude_sources;
std::string empty;
diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h
index 08b15a2..01fa278 100644
--- a/core/include/attachments/nginx_attachment_common.h
+++ b/core/include/attachments/nginx_attachment_common.h
@@ -64,6 +64,7 @@ typedef enum ngx_http_chunk_type
RESPONSE_END,
CONTENT_LENGTH,
METRIC_DATA_FROM_PLUGIN,
+ HOLD_DATA,
COUNT
} ngx_http_chunk_type_e;
@@ -85,6 +86,7 @@ typedef enum ngx_http_plugin_metric_type
IRRELEVANT_VERDICTS_COUNT,
RECONF_VERDICTS_COUNT,
INSPECT_VERDICTS_COUNT,
+ HOLD_VERDICTS_COUNT,
AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT,
MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT,
MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT,
@@ -103,6 +105,7 @@ typedef enum ngx_http_plugin_metric_type
MIN_REQ_BODY_SIZE_UPON_TIMEOUT,
RES_HEADER_THREAD_TIMEOUT,
RES_BODY_THREAD_TIMEOUT,
+ HOLD_THREAD_TIMEOUT,
AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT,
MAX_RES_BODY_SIZE_UPON_TIMEOUT,
MIN_RES_BODY_SIZE_UPON_TIMEOUT,
@@ -139,7 +142,8 @@ typedef enum ngx_http_cp_verdict
TRAFFIC_VERDICT_DROP,
TRAFFIC_VERDICT_INJECT,
TRAFFIC_VERDICT_IRRELEVANT,
- TRAFFIC_VERDICT_RECONF
+ TRAFFIC_VERDICT_RECONF,
+ TRAFFIC_VERDICT_WAIT
} ngx_http_cp_verdict_e;
#ifdef __cplusplus
diff --git a/core/include/attachments/nginx_attachment_util.h b/core/include/attachments/nginx_attachment_util.h
index 3532ca0..b3240f4 100644
--- a/core/include/attachments/nginx_attachment_util.h
+++ b/core/include/attachments/nginx_attachment_util.h
@@ -33,12 +33,15 @@ ngx_http_inspection_mode_e getInspectionMode();
unsigned int getNumOfNginxIpcElements();
unsigned int getKeepAliveIntervalMsec();
unsigned int getDbgLevel();
-int isDebugContext(c_str client, c_str server, unsigned int port, c_str method, c_str host , c_str uri);
+int isDebugContext(c_str client, c_str server, unsigned int port, c_str method, c_str host, c_str uri);
c_str getStaticResourcesPath();
int isFailOpenMode();
unsigned int getFailOpenTimeout();
+int isFailOpenHoldMode();
+unsigned int getFailOpenHoldTimeout();
+
unsigned int getMaxSessionsPerMinute();
int isFailOpenOnSessionLimit();
@@ -52,6 +55,8 @@ unsigned int getResProccessingTimeout();
unsigned int getResHeaderThreadTimeout();
unsigned int getResBodyThreadTimeout();
+unsigned int getWaitingForVerdictThreadTimeout();
+
int isIPAddress(c_str ip_str);
int isSkipSource(c_str ip_str);
diff --git a/core/shmem_ipc/shared_ring_queue.c b/core/shmem_ipc/shared_ring_queue.c
index 53f5c54..8f152e1 100644
--- a/core/shmem_ipc/shared_ring_queue.c
+++ b/core/shmem_ipc/shared_ring_queue.c
@@ -26,8 +26,9 @@
#include "shared_ipc_debug.h"
-static const uint16_t empty_buff_mgmt_magic = 0xcafe;
-static const uint16_t skip_buff_mgmt_magic = 0xbeef;
+static const uint16_t empty_buff_mgmt_magic = 0xfffe;
+static const uint16_t skip_buff_mgmt_magic = 0xfffd;
+static const uint32_t max_write_size = 0xfffc;
const uint16_t max_num_of_data_segments = sizeof(DataSegment)/sizeof(uint16_t);
char g_rx_location_name[MAX_ONE_WAY_QUEUE_NAME_LENGTH] = "";
@@ -52,11 +53,8 @@ getNumOfDataSegmentsNeeded(uint16_t data_size)
}
static int
-isThereEnoughMemoryInQueue(SharedRingQueue *queue, uint8_t num_of_elem_to_push)
+isThereEnoughMemoryInQueue(uint16_t write_pos, uint16_t read_pos, uint8_t num_of_elem_to_push)
{
- uint16_t write_pos = queue->write_pos;
- uint16_t read_pos = queue->read_pos;
- uint16_t num_of_data_segments = queue->num_of_data_segments;
int res;
writeDebug(
@@ -65,21 +63,21 @@ isThereEnoughMemoryInQueue(SharedRingQueue *queue, uint8_t num_of_elem_to_push)
num_of_elem_to_push,
write_pos,
read_pos,
- num_of_data_segments
+ g_num_of_data_segments
);
- if (num_of_elem_to_push >= num_of_data_segments) {
+ if (num_of_elem_to_push >= g_num_of_data_segments) {
writeDebug(TraceLevel, "Amount of elements to push is larger then amount of available elements in the queue");
return 0;
}
// add skipped elements during write that does not fit from cur write position till end of queue
- if (write_pos + num_of_elem_to_push > num_of_data_segments) {
- num_of_elem_to_push += num_of_data_segments - write_pos;
+ if (write_pos + num_of_elem_to_push > g_num_of_data_segments) {
+ num_of_elem_to_push += g_num_of_data_segments - write_pos;
}
// removing the aspect of circularity in queue and simulating as if the queue continued at its end
- if (write_pos + num_of_elem_to_push >= num_of_data_segments) {
- read_pos += num_of_data_segments;
+ if (write_pos + num_of_elem_to_push >= g_num_of_data_segments) {
+ read_pos += g_num_of_data_segments;
}
res = write_pos + num_of_elem_to_push < read_pos || write_pos >= read_pos;
@@ -87,6 +85,22 @@ isThereEnoughMemoryInQueue(SharedRingQueue *queue, uint8_t num_of_elem_to_push)
return res;
}
+static int
+isGetPossitionSucceccful(SharedRingQueue *queue, uint16_t *read_pos, uint16_t *write_pos)
+{
+ if (g_num_of_data_segments == 0) return 0;
+
+ *read_pos = queue->read_pos;
+ *write_pos = queue->write_pos;
+
+ if (queue->num_of_data_segments != g_num_of_data_segments) return 0;
+ if (queue->size_of_memory != g_memory_size) return 0;
+ if (*read_pos > g_num_of_data_segments) return 0;
+ if (*write_pos > g_num_of_data_segments) return 0;
+
+ return 1;
+}
+
void
resetRingQueue(SharedRingQueue *queue, uint16_t num_of_data_segments)
{
@@ -126,13 +140,14 @@ createSharedRingQueue(const char *shared_location_name, uint16_t num_of_data_seg
g_num_of_data_segments = num_of_data_segments;
- fd = shm_open(shared_location_name, shmem_fd_flags, S_IRWXU | S_IRWXG | S_IRWXO);
+ fd = shm_open(shared_location_name, shmem_fd_flags, S_IRUSR | S_IWUSR);
if (fd == -1) {
writeDebug(
WarningLevel,
- "createSharedRingQueue: Failed to open shared memory for '%s'. Errno: %d\n",
+ "createSharedRingQueue: Failed to open shared memory for '%s'. Errno: %d (%s)\n",
shared_location_name,
- errno
+ errno,
+ strerror(errno)
);
return NULL;
}
@@ -257,7 +272,7 @@ dumpRingQueueShmem(SharedRingQueue *queue)
writeDebug(WarningLevel, "mgmt_segment:");
buffer_mgmt = (uint16_t *)queue->mgmt_segment.data;
- for (segment_idx = 0; segment_idx < max_num_of_data_segments; segment_idx++) {
+ for (segment_idx = 0; segment_idx < queue->num_of_data_segments; segment_idx++) {
writeDebug(WarningLevel, "%s%u", (segment_idx == 0 ? " " : ", "), buffer_mgmt[segment_idx]);
}
@@ -275,39 +290,44 @@ dumpRingQueueShmem(SharedRingQueue *queue)
int
peekToQueue(SharedRingQueue *queue, const char **output_buffer, uint16_t *output_buffer_size)
{
- uint16_t read_pos = queue->read_pos;
- const uint16_t num_of_data_segments = queue->num_of_data_segments;
+ uint16_t read_pos;
+ uint16_t write_pos;
uint16_t *buffer_mgmt = (uint16_t *)queue->mgmt_segment.data;
+ if (!isGetPossitionSucceccful(queue, &read_pos, &write_pos)) {
+ writeDebug(WarningLevel, "Corrupted shared memory - cannot peek");
+ return -1;
+ }
+
writeDebug(
TraceLevel,
"Reading data from queue. Read index: %u, number of queue elements: %u",
read_pos,
- num_of_data_segments
+ g_num_of_data_segments
);
- if (isQueueEmpty(queue)) {
+ if (read_pos == write_pos) {
writeDebug(WarningLevel, "peekToQueue: Failed to read from an empty queue\n");
return -1;
}
- if (read_pos >= num_of_data_segments) {
+ if (read_pos >= g_num_of_data_segments) {
writeDebug(
WarningLevel,
"peekToQueue: Failed to read from a corrupted queue! (read_pos= %d > num_of_data_segments=%d)\n",
read_pos,
- num_of_data_segments
+ g_num_of_data_segments
);
return CORRUPTED_SHMEM_ERROR;
}
if (buffer_mgmt[read_pos] == skip_buff_mgmt_magic) {
- for ( ; read_pos < num_of_data_segments && buffer_mgmt[read_pos] == skip_buff_mgmt_magic; ++read_pos) {
+ for ( ; read_pos < g_num_of_data_segments && buffer_mgmt[read_pos] == skip_buff_mgmt_magic; ++read_pos) {
buffer_mgmt[read_pos] = empty_buff_mgmt_magic;
}
}
- if (read_pos == num_of_data_segments) read_pos = 0;
+ if (read_pos == g_num_of_data_segments) read_pos = 0;
*output_buffer_size = buffer_mgmt[read_pos];
*output_buffer = queue->data_segment[read_pos].data;
@@ -332,25 +352,42 @@ pushBuffersToQueue(
)
{
int idx;
- const uint16_t num_of_queue_elem = queue->num_of_data_segments;
- uint16_t write_pos = queue->write_pos;
- uint16_t total_elem_size = 0;
+ uint32_t large_total_elem_size = 0;
+ uint16_t read_pos;
+ uint16_t write_pos;
+ uint16_t total_elem_size;
uint16_t *buffer_mgmt = (uint16_t *)queue->mgmt_segment.data;
uint16_t end_pos;
uint16_t num_of_segments_to_write;
char *current_copy_pos;
+ if (!isGetPossitionSucceccful(queue, &read_pos, &write_pos)) {
+ writeDebug(WarningLevel, "Corrupted shared memory - cannot push new buffers");
+ return -1;
+ }
+
writeDebug(
TraceLevel,
"Writing new data to queue. write index: %u, number of queue elements: %u, number of elements to push: %u",
write_pos,
- num_of_queue_elem,
+ g_num_of_data_segments,
num_of_input_buffers
);
for (idx = 0; idx < num_of_input_buffers; idx++) {
- total_elem_size += input_buffers_sizes[idx];
+ large_total_elem_size += input_buffers_sizes[idx];
+
+ if (large_total_elem_size > max_write_size) {
+ writeDebug(
+ WarningLevel,
+ "Requested write size %u exceeds the %u write limit",
+ large_total_elem_size,
+ max_write_size
+ );
+ return -1;
+ }
}
+ total_elem_size = (uint16_t)large_total_elem_size;
num_of_segments_to_write = getNumOfDataSegmentsNeeded(total_elem_size);
@@ -362,23 +399,23 @@ pushBuffersToQueue(
);
- if (!isThereEnoughMemoryInQueue(queue, num_of_segments_to_write)) {
+ if (!isThereEnoughMemoryInQueue(write_pos, read_pos, num_of_segments_to_write)) {
writeDebug(WarningLevel, "Cannot write to a full queue\n");
return -1;
}
- if (write_pos >= num_of_queue_elem) {
+ if (write_pos >= g_num_of_data_segments) {
writeDebug(
WarningLevel,
"Cannot write to a location outside the queue. Write index: %u, number of queue elements: %u",
write_pos,
- num_of_queue_elem
+ g_num_of_data_segments
);
return -1;
}
- if (write_pos + num_of_segments_to_write > num_of_queue_elem) {
- for ( ; write_pos < num_of_queue_elem; ++write_pos) {
+ if (write_pos + num_of_segments_to_write > g_num_of_data_segments) {
+ for ( ; write_pos < g_num_of_data_segments; ++write_pos) {
buffer_mgmt[write_pos] = skip_buff_mgmt_magic;
}
write_pos = 0;
@@ -411,8 +448,9 @@ pushBuffersToQueue(
buffer_mgmt[write_pos] = skip_buff_mgmt_magic;
}
- queue->write_pos = write_pos < num_of_queue_elem ? write_pos : 0;
- writeDebug(TraceLevel, "Successfully pushed data to queue. New write index: %u", queue->write_pos);
+ if (write_pos >= g_num_of_data_segments) write_pos = 0;
+ queue->write_pos = write_pos;
+ writeDebug(TraceLevel, "Successfully pushed data to queue. New write index: %u", write_pos);
return 0;
}
@@ -427,19 +465,24 @@ int
popFromQueue(SharedRingQueue *queue)
{
uint16_t num_of_read_segments;
- uint16_t read_pos = queue->read_pos;
+ uint16_t read_pos;
+ uint16_t write_pos;
uint16_t end_pos;
- uint16_t num_of_data_segments = queue->num_of_data_segments;
uint16_t *buffer_mgmt = (uint16_t *)queue->mgmt_segment.data;
+ if (!isGetPossitionSucceccful(queue, &read_pos, &write_pos)) {
+ writeDebug(WarningLevel, "Corrupted shared memory - cannot pop data");
+ return -1;
+ }
+
writeDebug(
TraceLevel,
"Removing data from queue. new data to queue. Read index: %u, number of queue elements: %u",
read_pos,
- num_of_data_segments
+ g_num_of_data_segments
);
- if (isQueueEmpty(queue)) {
+ if (read_pos == write_pos) {
writeDebug(TraceLevel, "Cannot pop data from empty queue");
return -1;
}
@@ -460,16 +503,16 @@ popFromQueue(SharedRingQueue *queue)
buffer_mgmt[read_pos] = empty_buff_mgmt_magic;
}
- if (read_pos < num_of_data_segments && buffer_mgmt[read_pos] == skip_buff_mgmt_magic) {
- for ( ; read_pos < num_of_data_segments; ++read_pos ) {
+ if (read_pos < g_num_of_data_segments && buffer_mgmt[read_pos] == skip_buff_mgmt_magic) {
+ for ( ; read_pos < g_num_of_data_segments; ++read_pos ) {
buffer_mgmt[read_pos] = empty_buff_mgmt_magic;
}
}
- if (read_pos == num_of_data_segments) read_pos = 0;
+ if (read_pos == g_num_of_data_segments) read_pos = 0;
queue->read_pos = read_pos;
- writeDebug(TraceLevel, "Successfully popped data from queue. New read index: %u", queue->read_pos);
+ writeDebug(TraceLevel, "Successfully popped data from queue. New read index: %u", read_pos);
return 0;
}
diff --git a/core/shmem_ipc/shmem_ipc.c b/core/shmem_ipc/shmem_ipc.c
index d004da6..e492fe6 100644
--- a/core/shmem_ipc/shmem_ipc.c
+++ b/core/shmem_ipc/shmem_ipc.c
@@ -50,6 +50,7 @@ debugInitial(int is_error, const char *func, const char *file, int line_num, con
va_start(args, fmt);
vprintf(fmt, args);
va_end(args);
+ printf("\n");
}
void (*debug_int)(int is_error, const char *func, const char *file, int line_num, const char *fmt, ...) = debugInitial;
@@ -100,7 +101,7 @@ createOneWayIPCQueue(
return NULL;
}
- if (is_owner && chmod(shmem_path, 0666) == -1) {
+ if (is_owner && chown(shmem_path, user_id, group_id) == -1) {
writeDebug(WarningLevel, "Failed to set the permissions");
destroySharedRingQueue(ring_queue, is_owner, isTowardsOwner(is_owner, is_tx_queue));
return NULL;
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
new file mode 100644
index 0000000..97e72b1
--- /dev/null
+++ b/docker/CMakeLists.txt
@@ -0,0 +1,11 @@
+install(TARGETS ngx_module DESTINATION lib)
+
+add_custom_command(
+ OUTPUT ${CMAKE_INSTALL_PREFIX}/nginx-docker.img
+ COMMAND docker build -t nginx-docker ${CMAKE_INSTALL_PREFIX}
+ COMMAND docker tag nginx-docker ${OUTPUT_DOCKER_IMAGE}
+ COMMAND docker push ${OUTPUT_DOCKER_IMAGE}
+ COMMAND docker image save nginx-docker -o ${CMAKE_INSTALL_PREFIX}/nginx-docker.img
+)
+
+add_custom_target(docker DEPENDS ${CMAKE_INSTALL_PREFIX}/nginx-docker.img)
diff --git a/docker/Dockerfile b/docker/Dockerfile
new file mode 100644
index 0000000..759018e
--- /dev/null
+++ b/docker/Dockerfile
@@ -0,0 +1,18 @@
+FROM
+
+USER root
+
+RUN apk update
+RUN apk add --no-cache -u busybox
+RUN apk add --no-cache -u zlib
+RUN apk add --no-cache libstdc++
+
+USER
+
+COPY lib/libosrc_compression_utils.so /usr/lib/libosrc_compression_utils.so
+COPY lib/libosrc_nginx_attachment_util.so /usr/lib/libosrc_nginx_attachment_util.so
+COPY lib/libosrc_shmem_ipc.so /usr/lib/libosrc_shmem_ipc.so
+COPY lib/libngx_module.so /usr/lib/nginx/modules/ngx_cp_attachment_module.so
+
+RUN echo "load_module /usr/lib/nginx/modules/ngx_cp_attachment_module.so;"|cat - /etc/nginx/nginx.conf> /tmp/out && mv /tmp/out /etc/nginx/nginx.conf
+RUN [ -f /etc/nginx/template/nginx.tmpl ] && echo "load_module /usr/lib/nginx/modules/ngx_cp_attachment_module.so;"|cat - /etc/nginx/template/nginx.tmpl> /tmp/out.tmpl && mv /tmp/out.tmpl /etc/nginx/template/nginx.tmpl
diff --git a/external/cereal/archives/json.hpp b/external/cereal/archives/json.hpp
index 872b811..f864fb5 100644
--- a/external/cereal/archives/json.hpp
+++ b/external/cereal/archives/json.hpp
@@ -44,6 +44,7 @@ namespace cereal
#ifndef CEREAL_RAPIDJSON_ASSERT
#define CEREAL_RAPIDJSON_ASSERT(x) if(!(x)){ \
throw ::cereal::RapidJSONException("rapidjson internal assertion failure: " #x); }
+#define RAPIDJSON_ASSERT(x) CEREAL_RAPIDJSON_ASSERT(x)
#endif // RAPIDJSON_ASSERT
// Enable support for parsing of nan, inf, -inf
diff --git a/external/cereal/external/rapidjson/allocators.h b/external/cereal/external/rapidjson/allocators.h
index 554dc4a..ddcf478 100644
--- a/external/cereal/external/rapidjson/allocators.h
+++ b/external/cereal/external/rapidjson/allocators.h
@@ -1,6 +1,6 @@
// Tencent is pleased to support the open source community by making RapidJSON available.
//
-// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
@@ -12,12 +12,20 @@
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
-#ifndef CEREAL_RAPIDJSON_ALLOCATORS_H_
-#define CEREAL_RAPIDJSON_ALLOCATORS_H_
+#ifndef RAPIDJSON_ALLOCATORS_H_
+#define RAPIDJSON_ALLOCATORS_H_
#include "rapidjson.h"
+#include "internal/meta.h"
-CEREAL_RAPIDJSON_NAMESPACE_BEGIN
+#include
+#include
+
+#if RAPIDJSON_HAS_CXX11
+#include
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
///////////////////////////////////////////////////////////////////////////////
// Allocator
@@ -52,6 +60,19 @@ concept Allocator {
\endcode
*/
+
+/*! \def RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY
+ \ingroup RAPIDJSON_CONFIG
+ \brief User-defined kDefaultChunkCapacity definition.
+
+ User can define this as any \c size that is a power of 2.
+*/
+
+#ifndef RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY
+#define RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY (64 * 1024)
+#endif
+
+
///////////////////////////////////////////////////////////////////////////////
// CrtAllocator
@@ -64,19 +85,26 @@ public:
static const bool kNeedFree = true;
void* Malloc(size_t size) {
if (size) // behavior of malloc(0) is implementation defined.
- return std::malloc(size);
+ return RAPIDJSON_MALLOC(size);
else
return NULL; // standardize to returning NULL.
}
void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) {
(void)originalSize;
if (newSize == 0) {
- std::free(originalPtr);
+ RAPIDJSON_FREE(originalPtr);
return NULL;
}
- return std::realloc(originalPtr, newSize);
+ return RAPIDJSON_REALLOC(originalPtr, newSize);
+ }
+ static void Free(void *ptr) RAPIDJSON_NOEXCEPT { RAPIDJSON_FREE(ptr); }
+
+ bool operator==(const CrtAllocator&) const RAPIDJSON_NOEXCEPT {
+ return true;
+ }
+ bool operator!=(const CrtAllocator&) const RAPIDJSON_NOEXCEPT {
+ return false;
}
- static void Free(void *ptr) { std::free(ptr); }
};
///////////////////////////////////////////////////////////////////////////////
@@ -100,16 +128,64 @@ public:
*/
template
class MemoryPoolAllocator {
+ //! Chunk header for perpending to each chunk.
+ /*! Chunks are stored as a singly linked list.
+ */
+ struct ChunkHeader {
+ size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself).
+ size_t size; //!< Current size of allocated memory in bytes.
+ ChunkHeader *next; //!< Next chunk in the linked list.
+ };
+
+ struct SharedData {
+ ChunkHeader *chunkHead; //!< Head of the chunk linked-list. Only the head chunk serves allocation.
+ BaseAllocator* ownBaseAllocator; //!< base allocator created by this object.
+ size_t refcount;
+ bool ownBuffer;
+ };
+
+ static const size_t SIZEOF_SHARED_DATA = RAPIDJSON_ALIGN(sizeof(SharedData));
+ static const size_t SIZEOF_CHUNK_HEADER = RAPIDJSON_ALIGN(sizeof(ChunkHeader));
+
+ static inline ChunkHeader *GetChunkHead(SharedData *shared)
+ {
+ return reinterpret_cast(reinterpret_cast(shared) + SIZEOF_SHARED_DATA);
+ }
+ static inline uint8_t *GetChunkBuffer(SharedData *shared)
+ {
+ return reinterpret_cast(shared->chunkHead) + SIZEOF_CHUNK_HEADER;
+ }
+
+ static const size_t kDefaultChunkCapacity = RAPIDJSON_ALLOCATOR_DEFAULT_CHUNK_CAPACITY; //!< Default chunk capacity.
+
public:
static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator)
+ static const bool kRefCounted = true; //!< Tell users that this allocator is reference counted on copy
//! Constructor with chunkSize.
/*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize.
\param baseAllocator The allocator for allocating memory chunks.
*/
+ explicit
MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) :
- chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+ chunk_capacity_(chunkSize),
+ baseAllocator_(baseAllocator ? baseAllocator : RAPIDJSON_NEW(BaseAllocator)()),
+ shared_(static_cast(baseAllocator_ ? baseAllocator_->Malloc(SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER) : 0))
{
+ RAPIDJSON_ASSERT(baseAllocator_ != 0);
+ RAPIDJSON_ASSERT(shared_ != 0);
+ if (baseAllocator) {
+ shared_->ownBaseAllocator = 0;
+ }
+ else {
+ shared_->ownBaseAllocator = baseAllocator_;
+ }
+ shared_->chunkHead = GetChunkHead(shared_);
+ shared_->chunkHead->capacity = 0;
+ shared_->chunkHead->size = 0;
+ shared_->chunkHead->next = 0;
+ shared_->ownBuffer = true;
+ shared_->refcount = 1;
}
//! Constructor with user-supplied buffer.
@@ -123,41 +199,101 @@ public:
\param baseAllocator The allocator for allocating memory chunks.
*/
MemoryPoolAllocator(void *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) :
- chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0)
+ chunk_capacity_(chunkSize),
+ baseAllocator_(baseAllocator),
+ shared_(static_cast(AlignBuffer(buffer, size)))
{
- CEREAL_RAPIDJSON_ASSERT(buffer != 0);
- CEREAL_RAPIDJSON_ASSERT(size > sizeof(ChunkHeader));
- chunkHead_ = reinterpret_cast(buffer);
- chunkHead_->capacity = size - sizeof(ChunkHeader);
- chunkHead_->size = 0;
- chunkHead_->next = 0;
+ RAPIDJSON_ASSERT(size >= SIZEOF_SHARED_DATA + SIZEOF_CHUNK_HEADER);
+ shared_->chunkHead = GetChunkHead(shared_);
+ shared_->chunkHead->capacity = size - SIZEOF_SHARED_DATA - SIZEOF_CHUNK_HEADER;
+ shared_->chunkHead->size = 0;
+ shared_->chunkHead->next = 0;
+ shared_->ownBaseAllocator = 0;
+ shared_->ownBuffer = false;
+ shared_->refcount = 1;
}
+ MemoryPoolAllocator(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT :
+ chunk_capacity_(rhs.chunk_capacity_),
+ baseAllocator_(rhs.baseAllocator_),
+ shared_(rhs.shared_)
+ {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
+ ++shared_->refcount;
+ }
+ MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) RAPIDJSON_NOEXCEPT
+ {
+ RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0);
+ ++rhs.shared_->refcount;
+ this->~MemoryPoolAllocator();
+ baseAllocator_ = rhs.baseAllocator_;
+ chunk_capacity_ = rhs.chunk_capacity_;
+ shared_ = rhs.shared_;
+ return *this;
+ }
+
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
+ MemoryPoolAllocator(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT :
+ chunk_capacity_(rhs.chunk_capacity_),
+ baseAllocator_(rhs.baseAllocator_),
+ shared_(rhs.shared_)
+ {
+ RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0);
+ rhs.shared_ = 0;
+ }
+ MemoryPoolAllocator& operator=(MemoryPoolAllocator&& rhs) RAPIDJSON_NOEXCEPT
+ {
+ RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0);
+ this->~MemoryPoolAllocator();
+ baseAllocator_ = rhs.baseAllocator_;
+ chunk_capacity_ = rhs.chunk_capacity_;
+ shared_ = rhs.shared_;
+ rhs.shared_ = 0;
+ return *this;
+ }
+#endif
+
//! Destructor.
/*! This deallocates all memory chunks, excluding the user-supplied buffer.
*/
- ~MemoryPoolAllocator() {
+ ~MemoryPoolAllocator() RAPIDJSON_NOEXCEPT {
+ if (!shared_) {
+ // do nothing if moved
+ return;
+ }
+ if (shared_->refcount > 1) {
+ --shared_->refcount;
+ return;
+ }
Clear();
- CEREAL_RAPIDJSON_DELETE(ownBaseAllocator_);
+ BaseAllocator *a = shared_->ownBaseAllocator;
+ if (shared_->ownBuffer) {
+ baseAllocator_->Free(shared_);
+ }
+ RAPIDJSON_DELETE(a);
}
- //! Deallocates all memory chunks, excluding the user-supplied buffer.
- void Clear() {
- while (chunkHead_ && chunkHead_ != userBuffer_) {
- ChunkHeader* next = chunkHead_->next;
- baseAllocator_->Free(chunkHead_);
- chunkHead_ = next;
+ //! Deallocates all memory chunks, excluding the first/user one.
+ void Clear() RAPIDJSON_NOEXCEPT {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
+ for (;;) {
+ ChunkHeader* c = shared_->chunkHead;
+ if (!c->next) {
+ break;
+ }
+ shared_->chunkHead = c->next;
+ baseAllocator_->Free(c);
}
- if (chunkHead_ && chunkHead_ == userBuffer_)
- chunkHead_->size = 0; // Clear user buffer
+ shared_->chunkHead->size = 0;
}
//! Computes the total capacity of allocated memory chunks.
/*! \return total capacity in bytes.
*/
- size_t Capacity() const {
+ size_t Capacity() const RAPIDJSON_NOEXCEPT {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
size_t capacity = 0;
- for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+ for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next)
capacity += c->capacity;
return capacity;
}
@@ -165,25 +301,35 @@ public:
//! Computes the memory blocks allocated.
/*! \return total used bytes.
*/
- size_t Size() const {
+ size_t Size() const RAPIDJSON_NOEXCEPT {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
size_t size = 0;
- for (ChunkHeader* c = chunkHead_; c != 0; c = c->next)
+ for (ChunkHeader* c = shared_->chunkHead; c != 0; c = c->next)
size += c->size;
return size;
}
+ //! Whether the allocator is shared.
+ /*! \return true or false.
+ */
+ bool Shared() const RAPIDJSON_NOEXCEPT {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
+ return shared_->refcount > 1;
+ }
+
//! Allocates a memory block. (concept Allocator)
void* Malloc(size_t size) {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
if (!size)
return NULL;
- size = CEREAL_RAPIDJSON_ALIGN(size);
- if (chunkHead_ == 0 || chunkHead_->size + size > chunkHead_->capacity)
+ size = RAPIDJSON_ALIGN(size);
+ if (RAPIDJSON_UNLIKELY(shared_->chunkHead->size + size > shared_->chunkHead->capacity))
if (!AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size))
return NULL;
- void *buffer = reinterpret_cast(chunkHead_) + CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size;
- chunkHead_->size += size;
+ void *buffer = GetChunkBuffer(shared_) + shared_->chunkHead->size;
+ shared_->chunkHead->size += size;
return buffer;
}
@@ -192,21 +338,22 @@ public:
if (originalPtr == 0)
return Malloc(newSize);
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
if (newSize == 0)
return NULL;
- originalSize = CEREAL_RAPIDJSON_ALIGN(originalSize);
- newSize = CEREAL_RAPIDJSON_ALIGN(newSize);
+ originalSize = RAPIDJSON_ALIGN(originalSize);
+ newSize = RAPIDJSON_ALIGN(newSize);
// Do not shrink if new size is smaller than original
if (originalSize >= newSize)
return originalPtr;
// Simply expand it if it is the last allocation and there is sufficient space
- if (originalPtr == reinterpret_cast(chunkHead_) + CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + chunkHead_->size - originalSize) {
+ if (originalPtr == GetChunkBuffer(shared_) + shared_->chunkHead->size - originalSize) {
size_t increment = static_cast(newSize - originalSize);
- if (chunkHead_->size + increment <= chunkHead_->capacity) {
- chunkHead_->size += increment;
+ if (shared_->chunkHead->size + increment <= shared_->chunkHead->capacity) {
+ shared_->chunkHead->size += increment;
return originalPtr;
}
}
@@ -222,50 +369,325 @@ public:
}
//! Frees a memory block (concept Allocator)
- static void Free(void *ptr) { (void)ptr; } // Do nothing
+ static void Free(void *ptr) RAPIDJSON_NOEXCEPT { (void)ptr; } // Do nothing
+
+ //! Compare (equality) with another MemoryPoolAllocator
+ bool operator==(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT {
+ RAPIDJSON_NOEXCEPT_ASSERT(shared_->refcount > 0);
+ RAPIDJSON_NOEXCEPT_ASSERT(rhs.shared_->refcount > 0);
+ return shared_ == rhs.shared_;
+ }
+ //! Compare (inequality) with another MemoryPoolAllocator
+ bool operator!=(const MemoryPoolAllocator& rhs) const RAPIDJSON_NOEXCEPT {
+ return !operator==(rhs);
+ }
private:
- //! Copy constructor is not permitted.
- MemoryPoolAllocator(const MemoryPoolAllocator& rhs) /* = delete */;
- //! Copy assignment operator is not permitted.
- MemoryPoolAllocator& operator=(const MemoryPoolAllocator& rhs) /* = delete */;
-
//! Creates a new chunk.
/*! \param capacity Capacity of the chunk in bytes.
\return true if success.
*/
bool AddChunk(size_t capacity) {
if (!baseAllocator_)
- ownBaseAllocator_ = baseAllocator_ = CEREAL_RAPIDJSON_NEW(BaseAllocator());
- if (ChunkHeader* chunk = reinterpret_cast(baseAllocator_->Malloc(CEREAL_RAPIDJSON_ALIGN(sizeof(ChunkHeader)) + capacity))) {
+ shared_->ownBaseAllocator = baseAllocator_ = RAPIDJSON_NEW(BaseAllocator)();
+ if (ChunkHeader* chunk = static_cast(baseAllocator_->Malloc(SIZEOF_CHUNK_HEADER + capacity))) {
chunk->capacity = capacity;
chunk->size = 0;
- chunk->next = chunkHead_;
- chunkHead_ = chunk;
+ chunk->next = shared_->chunkHead;
+ shared_->chunkHead = chunk;
return true;
}
else
return false;
}
- static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity.
+ static inline void* AlignBuffer(void* buf, size_t &size)
+ {
+ RAPIDJSON_NOEXCEPT_ASSERT(buf != 0);
+ const uintptr_t mask = sizeof(void*) - 1;
+ const uintptr_t ubuf = reinterpret_cast(buf);
+ if (RAPIDJSON_UNLIKELY(ubuf & mask)) {
+ const uintptr_t abuf = (ubuf + mask) & ~mask;
+ RAPIDJSON_ASSERT(size >= abuf - ubuf);
+ buf = reinterpret_cast(abuf);
+ size -= abuf - ubuf;
+ }
+ return buf;
+ }
- //! Chunk header for perpending to each chunk.
- /*! Chunks are stored as a singly linked list.
- */
- struct ChunkHeader {
- size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself).
- size_t size; //!< Current size of allocated memory in bytes.
- ChunkHeader *next; //!< Next chunk in the linked list.
- };
-
- ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation.
size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated.
- void *userBuffer_; //!< User supplied buffer.
BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks.
- BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object.
+ SharedData *shared_; //!< The shared data of the allocator
};
-CEREAL_RAPIDJSON_NAMESPACE_END
+namespace internal {
+ template
+ struct IsRefCounted :
+ public FalseType
+ { };
+ template
+ struct IsRefCounted::Type> :
+ public TrueType
+ { };
+}
-#endif // CEREAL_RAPIDJSON_ENCODINGS_H_
+template
+inline T* Realloc(A& a, T* old_p, size_t old_n, size_t new_n)
+{
+ RAPIDJSON_NOEXCEPT_ASSERT(old_n <= std::numeric_limits::max() / sizeof(T) && new_n <= std::numeric_limits::max() / sizeof(T));
+ return static_cast(a.Realloc(old_p, old_n * sizeof(T), new_n * sizeof(T)));
+}
+
+template
+inline T *Malloc(A& a, size_t n = 1)
+{
+ return Realloc(a, NULL, 0, n);
+}
+
+template
+inline void Free(A& a, T *p, size_t n = 1)
+{
+ static_cast(Realloc(a, p, n, 0));
+}
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++) // std::allocator can safely be inherited
+#endif
+
+template
+class StdAllocator :
+ public std::allocator
+{
+ typedef std::allocator allocator_type;
+#if RAPIDJSON_HAS_CXX11
+ typedef std::allocator_traits traits_type;
+#else
+ typedef allocator_type traits_type;
+#endif
+
+public:
+ typedef BaseAllocator BaseAllocatorType;
+
+ StdAllocator() RAPIDJSON_NOEXCEPT :
+ allocator_type(),
+ baseAllocator_()
+ { }
+
+ StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT :
+ allocator_type(rhs),
+ baseAllocator_(rhs.baseAllocator_)
+ { }
+
+ template
+ StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT :
+ allocator_type(rhs),
+ baseAllocator_(rhs.baseAllocator_)
+ { }
+
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
+ StdAllocator(StdAllocator&& rhs) RAPIDJSON_NOEXCEPT :
+ allocator_type(std::move(rhs)),
+ baseAllocator_(std::move(rhs.baseAllocator_))
+ { }
+#endif
+#if RAPIDJSON_HAS_CXX11
+ using propagate_on_container_move_assignment = std::true_type;
+ using propagate_on_container_swap = std::true_type;
+#endif
+
+ /* implicit */
+ StdAllocator(const BaseAllocator& allocator) RAPIDJSON_NOEXCEPT :
+ allocator_type(),
+ baseAllocator_(allocator)
+ { }
+
+ ~StdAllocator() RAPIDJSON_NOEXCEPT
+ { }
+
+ template
+ struct rebind {
+ typedef StdAllocator other;
+ };
+
+ typedef typename traits_type::size_type size_type;
+ typedef typename traits_type::difference_type difference_type;
+
+ typedef typename traits_type::value_type value_type;
+ typedef typename traits_type::pointer pointer;
+ typedef typename traits_type::const_pointer const_pointer;
+
+#if RAPIDJSON_HAS_CXX11
+
+ typedef typename std::add_lvalue_reference::type &reference;
+ typedef typename std::add_lvalue_reference::type>::type &const_reference;
+
+ pointer address(reference r) const RAPIDJSON_NOEXCEPT
+ {
+ return std::addressof(r);
+ }
+ const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT
+ {
+ return std::addressof(r);
+ }
+
+ size_type max_size() const RAPIDJSON_NOEXCEPT
+ {
+ return traits_type::max_size(*this);
+ }
+
+ template
+ void construct(pointer p, Args&&... args)
+ {
+ traits_type::construct(*this, p, std::forward(args)...);
+ }
+ void destroy(pointer p)
+ {
+ traits_type::destroy(*this, p);
+ }
+
+#else // !RAPIDJSON_HAS_CXX11
+
+ typedef typename allocator_type::reference reference;
+ typedef typename allocator_type::const_reference const_reference;
+
+ pointer address(reference r) const RAPIDJSON_NOEXCEPT
+ {
+ return allocator_type::address(r);
+ }
+ const_pointer address(const_reference r) const RAPIDJSON_NOEXCEPT
+ {
+ return allocator_type::address(r);
+ }
+
+ size_type max_size() const RAPIDJSON_NOEXCEPT
+ {
+ return allocator_type::max_size();
+ }
+
+ void construct(pointer p, const_reference r)
+ {
+ allocator_type::construct(p, r);
+ }
+ void destroy(pointer p)
+ {
+ allocator_type::destroy(p);
+ }
+
+#endif // !RAPIDJSON_HAS_CXX11
+
+ template
+ U* allocate(size_type n = 1, const void* = 0)
+ {
+ return RAPIDJSON_NAMESPACE::Malloc(baseAllocator_, n);
+ }
+ template
+ void deallocate(U* p, size_type n = 1)
+ {
+ RAPIDJSON_NAMESPACE::Free(baseAllocator_, p, n);
+ }
+
+ pointer allocate(size_type n = 1, const void* = 0)
+ {
+ return allocate(n);
+ }
+ void deallocate(pointer p, size_type n = 1)
+ {
+ deallocate(p, n);
+ }
+
+#if RAPIDJSON_HAS_CXX11
+ using is_always_equal = std::is_empty;
+#endif
+
+ template
+ bool operator==(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT
+ {
+ return baseAllocator_ == rhs.baseAllocator_;
+ }
+ template
+ bool operator!=(const StdAllocator& rhs) const RAPIDJSON_NOEXCEPT
+ {
+ return !operator==(rhs);
+ }
+
+ //! rapidjson Allocator concept
+ static const bool kNeedFree = BaseAllocator::kNeedFree;
+ static const bool kRefCounted = internal::IsRefCounted::Value;
+ void* Malloc(size_t size)
+ {
+ return baseAllocator_.Malloc(size);
+ }
+ void* Realloc(void* originalPtr, size_t originalSize, size_t newSize)
+ {
+ return baseAllocator_.Realloc(originalPtr, originalSize, newSize);
+ }
+ static void Free(void *ptr) RAPIDJSON_NOEXCEPT
+ {
+ BaseAllocator::Free(ptr);
+ }
+
+private:
+ template
+ friend class StdAllocator; // access to StdAllocator.*
+
+ BaseAllocator baseAllocator_;
+};
+
+#if !RAPIDJSON_HAS_CXX17 // std::allocator deprecated in C++17
+template
+class StdAllocator :
+ public std::allocator
+{
+ typedef std::allocator allocator_type;
+
+public:
+ typedef BaseAllocator BaseAllocatorType;
+
+ StdAllocator() RAPIDJSON_NOEXCEPT :
+ allocator_type(),
+ baseAllocator_()
+ { }
+
+ StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT :
+ allocator_type(rhs),
+ baseAllocator_(rhs.baseAllocator_)
+ { }
+
+ template
+ StdAllocator(const StdAllocator& rhs) RAPIDJSON_NOEXCEPT :
+ allocator_type(rhs),
+ baseAllocator_(rhs.baseAllocator_)
+ { }
+
+ /* implicit */
+ StdAllocator(const BaseAllocator& baseAllocator) RAPIDJSON_NOEXCEPT :
+ allocator_type(),
+ baseAllocator_(baseAllocator)
+ { }
+
+ ~StdAllocator() RAPIDJSON_NOEXCEPT
+ { }
+
+ template
+ struct rebind {
+ typedef StdAllocator other;
+ };
+
+ typedef typename allocator_type::value_type value_type;
+
+private:
+ template
+ friend class StdAllocator; // access to StdAllocator.*
+
+ BaseAllocator baseAllocator_;
+};
+#endif
+
+#ifdef __GNUC__
+RAPIDJSON_DIAG_POP
+#endif
+
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_ENCODINGS_H_
diff --git a/external/cereal/external/rapidjson/cursorstreamwrapper.h b/external/cereal/external/rapidjson/cursorstreamwrapper.h
new file mode 100644
index 0000000..fd6513d
--- /dev/null
+++ b/external/cereal/external/rapidjson/cursorstreamwrapper.h
@@ -0,0 +1,78 @@
+// Tencent is pleased to support the open source community by making RapidJSON available.
+//
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
+//
+// Licensed under the MIT License (the "License"); you may not use this file except
+// in compliance with the License. You may obtain a copy of the License at
+//
+// http://opensource.org/licenses/MIT
+//
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+#ifndef RAPIDJSON_CURSORSTREAMWRAPPER_H_
+#define RAPIDJSON_CURSORSTREAMWRAPPER_H_
+
+#include "stream.h"
+
+#if defined(__GNUC__)
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(effc++)
+#endif
+
+#if defined(_MSC_VER) && _MSC_VER <= 1800
+RAPIDJSON_DIAG_PUSH
+RAPIDJSON_DIAG_OFF(4702) // unreachable code
+RAPIDJSON_DIAG_OFF(4512) // assignment operator could not be generated
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
+
+
+//! Cursor stream wrapper for counting line and column number if error exists.
+/*!
+ \tparam InputStream Any stream that implements Stream Concept
+*/
+template >
+class CursorStreamWrapper : public GenericStreamWrapper {
+public:
+ typedef typename Encoding::Ch Ch;
+
+ CursorStreamWrapper(InputStream& is):
+ GenericStreamWrapper(is), line_(1), col_(0) {}
+
+ // counting line and column number
+ Ch Take() {
+ Ch ch = this->is_.Take();
+ if(ch == '\n') {
+ line_ ++;
+ col_ = 0;
+ } else {
+ col_ ++;
+ }
+ return ch;
+ }
+
+ //! Get the error line number, if error exists.
+ size_t GetLine() const { return line_; }
+ //! Get the error column number, if error exists.
+ size_t GetColumn() const { return col_; }
+
+private:
+ size_t line_; //!< Current Line
+ size_t col_; //!< Current Column
+};
+
+#if defined(_MSC_VER) && _MSC_VER <= 1800
+RAPIDJSON_DIAG_POP
+#endif
+
+#if defined(__GNUC__)
+RAPIDJSON_DIAG_POP
+#endif
+
+RAPIDJSON_NAMESPACE_END
+
+#endif // RAPIDJSON_CURSORSTREAMWRAPPER_H_
diff --git a/external/cereal/external/rapidjson/document.h b/external/cereal/external/rapidjson/document.h
index 46faeb1..4f1e246 100644
--- a/external/cereal/external/rapidjson/document.h
+++ b/external/cereal/external/rapidjson/document.h
@@ -1,6 +1,6 @@
// Tencent is pleased to support the open source community by making RapidJSON available.
//
-// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved.
+// Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip.
//
// Licensed under the MIT License (the "License"); you may not use this file except
// in compliance with the License. You may obtain a copy of the License at
@@ -12,8 +12,8 @@
// CONDITIONS OF ANY KIND, either express or implied. See the License for the
// specific language governing permissions and limitations under the License.
-#ifndef CEREAL_RAPIDJSON_DOCUMENT_H_
-#define CEREAL_RAPIDJSON_DOCUMENT_H_
+#ifndef RAPIDJSON_DOCUMENT_H_
+#define RAPIDJSON_DOCUMENT_H_
/*! \file document.h */
@@ -24,35 +24,42 @@
#include "encodedstream.h"
#include // placement new
#include
-
-CEREAL_RAPIDJSON_DIAG_PUSH
-#ifdef _MSC_VER
-CEREAL_RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant
-CEREAL_RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data
+#ifdef __cpp_lib_three_way_comparison
+#include
#endif
+RAPIDJSON_DIAG_PUSH
#ifdef __clang__
-CEREAL_RAPIDJSON_DIAG_OFF(padded)
-CEREAL_RAPIDJSON_DIAG_OFF(switch-enum)
-CEREAL_RAPIDJSON_DIAG_OFF(c++98-compat)
+RAPIDJSON_DIAG_OFF(padded)
+RAPIDJSON_DIAG_OFF(switch-enum)
+RAPIDJSON_DIAG_OFF(c++98-compat)
+#elif defined(_MSC_VER)
+RAPIDJSON_DIAG_OFF(4127) // conditional expression is constant
+RAPIDJSON_DIAG_OFF(4244) // conversion from kXxxFlags to 'uint16_t', possible loss of data
#endif
#ifdef __GNUC__
-CEREAL_RAPIDJSON_DIAG_OFF(effc++)
-#if __GNUC__ >= 6
-CEREAL_RAPIDJSON_DIAG_OFF(terminate) // ignore throwing CEREAL_RAPIDJSON_ASSERT in CEREAL_RAPIDJSON_NOEXCEPT functions
-#endif
+RAPIDJSON_DIAG_OFF(effc++)
#endif // __GNUC__
-#ifndef CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS
-#include // std::iterator, std::random_access_iterator_tag
+#ifdef GetObject
+// see https://github.com/Tencent/rapidjson/issues/1448
+// a former included windows.h might have defined a macro called GetObject, which affects
+// GetObject defined here. This ensures the macro does not get applied
+#pragma push_macro("GetObject")
+#define RAPIDJSON_WINDOWS_GETOBJECT_WORKAROUND_APPLIED
+#undef GetObject
#endif
-#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS
-#include // std::move
+#ifndef RAPIDJSON_NOMEMBERITERATORCLASS
+#include // std::random_access_iterator_tag
#endif
-CEREAL_RAPIDJSON_NAMESPACE_BEGIN
+#if RAPIDJSON_USE_MEMBERSMAP
+#include // std::multimap
+#endif
+
+RAPIDJSON_NAMESPACE_BEGIN
// Forward declaration.
template
@@ -61,6 +68,48 @@ class GenericValue;
template
class GenericDocument;
+/*! \def RAPIDJSON_DEFAULT_ALLOCATOR
+ \ingroup RAPIDJSON_CONFIG
+ \brief Allows to choose default allocator.
+
+ User can define this to use CrtAllocator or MemoryPoolAllocator.
+*/
+#ifndef RAPIDJSON_DEFAULT_ALLOCATOR
+#define RAPIDJSON_DEFAULT_ALLOCATOR ::RAPIDJSON_NAMESPACE::MemoryPoolAllocator<::RAPIDJSON_NAMESPACE::CrtAllocator>
+#endif
+
+/*! \def RAPIDJSON_DEFAULT_STACK_ALLOCATOR
+ \ingroup RAPIDJSON_CONFIG
+ \brief Allows to choose default stack allocator for Document.
+
+ User can define this to use CrtAllocator or MemoryPoolAllocator.
+*/
+#ifndef RAPIDJSON_DEFAULT_STACK_ALLOCATOR
+#define RAPIDJSON_DEFAULT_STACK_ALLOCATOR ::RAPIDJSON_NAMESPACE::CrtAllocator
+#endif
+
+/*! \def RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY
+ \ingroup RAPIDJSON_CONFIG
+ \brief User defined kDefaultObjectCapacity value.
+
+ User can define this as any natural number.
+*/
+#ifndef RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY
+// number of objects that rapidjson::Value allocates memory for by default
+#define RAPIDJSON_VALUE_DEFAULT_OBJECT_CAPACITY 16
+#endif
+
+/*! \def RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY
+ \ingroup RAPIDJSON_CONFIG
+ \brief User defined kDefaultArrayCapacity value.
+
+ User can define this as any natural number.
+*/
+#ifndef RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY
+// number of array elements that rapidjson::Value allocates memory for by default
+#define RAPIDJSON_VALUE_DEFAULT_ARRAY_CAPACITY 16
+#endif
+
//! Name-value pair in a JSON object value.
/*!
This class was internal to GenericValue. It used to be a inner struct.
@@ -68,15 +117,51 @@ class GenericDocument;
https://code.google.com/p/rapidjson/issues/detail?id=64
*/
template
-struct GenericMember {
+class GenericMember {
+public:
GenericValue name; //!< name of member (must be a string)
GenericValue value; //!< value of member.
+
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
+ //! Move constructor in C++11
+ GenericMember(GenericMember&& rhs) RAPIDJSON_NOEXCEPT
+ : name(std::move(rhs.name)),
+ value(std::move(rhs.value))
+ {
+ }
+
+ //! Move assignment in C++11
+ GenericMember& operator=(GenericMember&& rhs) RAPIDJSON_NOEXCEPT {
+ return *this = static_cast(rhs);
+ }
+#endif
+
+ //! Assignment with move semantics.
+ /*! \param rhs Source of the assignment. Its name and value will become a null value after assignment.
+ */
+ GenericMember& operator=(GenericMember& rhs) RAPIDJSON_NOEXCEPT {
+ if (RAPIDJSON_LIKELY(this != &rhs)) {
+ name = rhs.name;
+ value = rhs.value;
+ }
+ return *this;
+ }
+
+ // swap() for std::sort() and other potential use in STL.
+ friend inline void swap(GenericMember& a, GenericMember& b) RAPIDJSON_NOEXCEPT {
+ a.name.Swap(b.name);
+ a.value.Swap(b.value);
+ }
+
+private:
+ //! Copy constructor is not permitted.
+ GenericMember(const GenericMember& rhs);
};
///////////////////////////////////////////////////////////////////////////////
// GenericMemberIterator
-#ifndef CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS
+#ifndef RAPIDJSON_NOMEMBERITERATORCLASS
//! (Constant) member iterator for a JSON object value
/*!
@@ -91,23 +176,20 @@ struct GenericMember {
conversions from iterator values to \c NULL,
e.g. from GenericValue::FindMember.
- \note Define \c CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a
+ \note Define \c RAPIDJSON_NOMEMBERITERATORCLASS to fall back to a
pointer-based implementation, if your platform doesn't provide
the C++ header.
\see GenericMember, GenericValue::MemberIterator, GenericValue::ConstMemberIterator
*/
template
-class GenericMemberIterator
- : public std::iterator >::Type> {
+class GenericMemberIterator {
friend class GenericValue;
template friend class GenericMemberIterator;
typedef GenericMember PlainType;
typedef typename internal::MaybeAddConst::Type ValueType;
- typedef std::iterator BaseType;
public:
//! Iterator type itself
@@ -117,12 +199,21 @@ public:
//! Non-constant iterator type
typedef GenericMemberIterator NonConstIterator;
+ /** \name std::iterator_traits support */
+ //@{
+ typedef ValueType value_type;
+ typedef ValueType * pointer;
+ typedef ValueType & reference;
+ typedef std::ptrdiff_t difference_type;
+ typedef std::random_access_iterator_tag iterator_category;
+ //@}
+
//! Pointer to (const) GenericMember
- typedef typename BaseType::pointer Pointer;
+ typedef pointer Pointer;
//! Reference to (const) GenericMember
- typedef typename BaseType::reference Reference;
+ typedef reference Reference;
//! Signed integer type (e.g. \c ptrdiff_t)
- typedef typename BaseType::difference_type DifferenceType;
+ typedef difference_type DifferenceType;
//! Default constructor (singular value)
/*! Creates an iterator pointing to no element.
@@ -168,12 +259,16 @@ public:
//! @name relations
//@{
- bool operator==(ConstIterator that) const { return ptr_ == that.ptr_; }
- bool operator!=(ConstIterator that) const { return ptr_ != that.ptr_; }
- bool operator<=(ConstIterator that) const { return ptr_ <= that.ptr_; }
- bool operator>=(ConstIterator that) const { return ptr_ >= that.ptr_; }
- bool operator< (ConstIterator that) const { return ptr_ < that.ptr_; }
- bool operator> (ConstIterator that) const { return ptr_ > that.ptr_; }
+ template bool operator==(const GenericMemberIterator& that) const { return ptr_ == that.ptr_; }
+ template bool operator!=(const GenericMemberIterator& that) const { return ptr_ != that.ptr_; }
+ template bool operator<=(const GenericMemberIterator& that) const { return ptr_ <= that.ptr_; }
+ template bool operator>=(const GenericMemberIterator& that) const { return ptr_ >= that.ptr_; }
+ template bool operator< (const GenericMemberIterator& that) const { return ptr_ < that.ptr_; }
+ template bool operator> (const GenericMemberIterator& that) const { return ptr_ > that.ptr_; }
+
+#ifdef __cpp_lib_three_way_comparison
+ template std::strong_ordering operator<=>(const GenericMemberIterator& that) const { return ptr_ <=> that.ptr_; }
+#endif
//@}
//! @name dereference
@@ -193,27 +288,29 @@ private:
Pointer ptr_; //!< raw pointer
};
-#else // CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS
+#else // RAPIDJSON_NOMEMBERITERATORCLASS
// class-based member iterator implementation disabled, use plain pointers
template
-struct GenericMemberIterator;
+class GenericMemberIterator;
//! non-const GenericMemberIterator
template
-struct GenericMemberIterator {
+class GenericMemberIterator {
+public:
//! use plain pointer as iterator type
typedef GenericMember* Iterator;
};
//! const GenericMemberIterator
template
-struct GenericMemberIterator {
+class GenericMemberIterator {
+public:
//! use plain const pointer as iterator type
typedef const GenericMember* Iterator;
};
-#endif // CEREAL_RAPIDJSON_NOMEMBERITERATORCLASS
+#endif // RAPIDJSON_NOMEMBERITERATORCLASS
///////////////////////////////////////////////////////////////////////////////
// GenericStringRef
@@ -275,7 +372,7 @@ struct GenericStringRef {
*/
#endif
template
- GenericStringRef(const CharType (&str)[N]) CEREAL_RAPIDJSON_NOEXCEPT
+ GenericStringRef(const CharType (&str)[N]) RAPIDJSON_NOEXCEPT
: s(str), length(N-1) {}
//! Explicitly create string reference from \c const character pointer
@@ -300,7 +397,7 @@ struct GenericStringRef {
*/
#endif
explicit GenericStringRef(const CharType* str)
- : s(str), length(internal::StrLen(str)){ CEREAL_RAPIDJSON_ASSERT(s != 0); }
+ : s(str), length(NotNullStrLen(str)) {}
//! Create constant string reference from pointer and length
#ifndef __clang__ // -Wdocumentation
@@ -312,12 +409,10 @@ struct GenericStringRef {
*/
#endif
GenericStringRef(const CharType* str, SizeType len)
- : s(str), length(len) { CEREAL_RAPIDJSON_ASSERT(s != 0); }
+ : s(RAPIDJSON_LIKELY(str) ? str : emptyString), length(len) { RAPIDJSON_ASSERT(str != 0 || len == 0u); }
GenericStringRef(const GenericStringRef& rhs) : s(rhs.s), length(rhs.length) {}
- GenericStringRef& operator=(const GenericStringRef& rhs) { s = rhs.s; length = rhs.length; }
-
//! implicit conversion to plain CharType pointer
operator const Ch *() const { return s; }
@@ -325,11 +420,24 @@ struct GenericStringRef {
const SizeType length; //!< length of the string (excluding the trailing NULL terminator)
private:
+ SizeType NotNullStrLen(const CharType* str) {
+ RAPIDJSON_ASSERT(str != 0);
+ return internal::StrLen(str);
+ }
+
+ /// Empty string - used when passing in a NULL pointer
+ static const Ch emptyString[];
+
//! Disallow construction from non-const array
template
GenericStringRef(CharType (&str)[N]) /* = delete */;
+ //! Copy assignment operator not permitted - immutable type
+ GenericStringRef& operator=(const GenericStringRef& rhs) /* = delete */;
};
+template
+const CharType GenericStringRef::emptyString[] = { CharType() };
+
//! Mark a character pointer as constant string
/*! Mark a plain character pointer as a "string literal". This function
can be used to avoid copying a character string to be referenced as a
@@ -344,7 +452,7 @@ private:
*/
template
inline GenericStringRef StringRef(const CharType* str) {
- return GenericStringRef(str, internal::StrLen(str));
+ return GenericStringRef(str);
}
//! Mark a character pointer as constant string
@@ -367,7 +475,7 @@ inline GenericStringRef StringRef(const CharType* str, size_t length)
return GenericStringRef(str, SizeType(length));
}
-#if CEREAL_RAPIDJSON_HAS_STDSTRING
+#if RAPIDJSON_HAS_STDSTRING
//! Mark a string object as constant string
/*! Mark a string object (e.g. \c std::string) as a "string literal".
This function can be used to avoid copying a string to be referenced as a
@@ -378,7 +486,7 @@ inline GenericStringRef StringRef(const CharType* str, size_t length)
\param str Constant string, lifetime assumed to be longer than the use of the string in e.g. a GenericValue
\return GenericStringRef string reference object
\relatesalso GenericStringRef
- \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING.
+ \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING.
*/
template
inline GenericStringRef StringRef(const std::basic_string& str) {
@@ -434,6 +542,26 @@ struct TypeHelper {
static ValueType& Set(ValueType& v, unsigned data, typename ValueType::AllocatorType&) { return v.SetUint(data); }
};
+#ifdef _MSC_VER
+RAPIDJSON_STATIC_ASSERT(sizeof(long) == sizeof(int));
+template
+struct TypeHelper {
+ static bool Is(const ValueType& v) { return v.IsInt(); }
+ static long Get(const ValueType& v) { return v.GetInt(); }
+ static ValueType& Set(ValueType& v, long data) { return v.SetInt(data); }
+ static ValueType& Set(ValueType& v, long data, typename ValueType::AllocatorType&) { return v.SetInt(data); }
+};
+
+RAPIDJSON_STATIC_ASSERT(sizeof(unsigned long) == sizeof(unsigned));
+template
+struct TypeHelper {
+ static bool Is(const ValueType& v) { return v.IsUint(); }
+ static unsigned long Get(const ValueType& v) { return v.GetUint(); }
+ static ValueType& Set(ValueType& v, unsigned long data) { return v.SetUint(data); }
+ static ValueType& Set(ValueType& v, unsigned long data, typename ValueType::AllocatorType&) { return v.SetUint(data); }
+};
+#endif
+
template
struct TypeHelper {
static bool Is(const ValueType& v) { return v.IsInt64(); }
@@ -475,7 +603,7 @@ struct TypeHelper {
static ValueType& Set(ValueType& v, const StringType data, typename ValueType::AllocatorType& a) { return v.SetString(data, a); }
};
-#if CEREAL_RAPIDJSON_HAS_STDSTRING
+#if RAPIDJSON_HAS_STDSTRING
template
struct TypeHelper > {
typedef std::basic_string StringType;
@@ -507,7 +635,7 @@ struct TypeHelper {
static bool Is(const ValueType& v) { return v.IsObject(); }
static ObjectType Get(ValueType& v) { return v.GetObject(); }
static ValueType& Set(ValueType& v, ObjectType data) { return v = data; }
- static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { v = data; }
+ static ValueType& Set(ValueType& v, ObjectType data, typename ValueType::AllocatorType&) { return v = data; }
};
template
@@ -536,7 +664,7 @@ template class GenericObject;
\tparam Encoding Encoding of the value. (Even non-string values need to have the same encoding in a document)
\tparam Allocator Allocator type for allocating memory of object, array and string.
*/
-template >
+template
class GenericValue {
public:
//! Name-value pair in an object.
@@ -559,11 +687,11 @@ public:
//@{
//! Default constructor creates a null value.
- GenericValue() CEREAL_RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; }
+ GenericValue() RAPIDJSON_NOEXCEPT : data_() { data_.f.flags = kNullFlag; }
-#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
//! Move constructor in C++11
- GenericValue(GenericValue&& rhs) CEREAL_RAPIDJSON_NOEXCEPT : data_(rhs.data_) {
+ GenericValue(GenericValue&& rhs) RAPIDJSON_NOEXCEPT : data_(rhs.data_) {
rhs.data_.f.flags = kNullFlag; // give up contents
}
#endif
@@ -572,7 +700,7 @@ private:
//! Copy constructor is not permitted.
GenericValue(const GenericValue& rhs);
-#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
//! Moving from a GenericDocument is not permitted.
template
GenericValue(GenericDocument&& rhs);
@@ -589,12 +717,12 @@ public:
\param type Type of the value.
\note Default content for number is zero.
*/
- explicit GenericValue(Type type) CEREAL_RAPIDJSON_NOEXCEPT : data_() {
- static const uint16_t defaultFlags[7] = {
+ explicit GenericValue(Type type) RAPIDJSON_NOEXCEPT : data_() {
+ static const uint16_t defaultFlags[] = {
kNullFlag, kFalseFlag, kTrueFlag, kObjectFlag, kArrayFlag, kShortStringFlag,
kNumberAnyFlag
};
- CEREAL_RAPIDJSON_ASSERT(type <= kNumberType);
+ RAPIDJSON_NOEXCEPT_ASSERT(type >= kNullType && type <= kNumberType);
data_.f.flags = defaultFlags[type];
// Use ShortString to store empty string.
@@ -607,10 +735,40 @@ public:
\tparam SourceAllocator allocator of \c rhs
\param rhs Value to copy from (read-only)
\param allocator Allocator for allocating copied elements and buffers. Commonly use GenericDocument::GetAllocator().
+ \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer)
\see CopyFrom()
*/
- template< typename SourceAllocator >
- GenericValue(const GenericValue& rhs, Allocator & allocator);
+ template
+ GenericValue(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) {
+ switch (rhs.GetType()) {
+ case kObjectType:
+ DoCopyMembers(rhs, allocator, copyConstStrings);
+ break;
+ case kArrayType: {
+ SizeType count = rhs.data_.a.size;
+ GenericValue* le = reinterpret_cast(allocator.Malloc(count * sizeof(GenericValue)));
+ const GenericValue* re = rhs.GetElementsPointer();
+ for (SizeType i = 0; i < count; i++)
+ new (&le[i]) GenericValue(re[i], allocator, copyConstStrings);
+ data_.f.flags = kArrayFlag;
+ data_.a.size = data_.a.capacity = count;
+ SetElementsPointer(le);
+ }
+ break;
+ case kStringType:
+ if (rhs.data_.f.flags == kConstStringFlag && !copyConstStrings) {
+ data_.f.flags = rhs.data_.f.flags;
+ data_ = *reinterpret_cast(&rhs.data_);
+ }
+ else
+ SetStringRaw(StringRef(rhs.GetString(), rhs.GetStringLength()), allocator);
+ break;
+ default:
+ data_.f.flags = rhs.data_.f.flags;
+ data_ = *reinterpret_cast(&rhs.data_);
+ break;
+ }
+ }
//! Constructor for boolean value.
/*! \param b Boolean value
@@ -618,65 +776,68 @@ public:
implicitly converted types like arbitrary pointers. Use an explicit cast
to \c bool, if you want to construct a boolean JSON value in such cases.
*/
-#ifndef CEREAL_RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen
+#ifndef RAPIDJSON_DOXYGEN_RUNNING // hide SFINAE from Doxygen
template
- explicit GenericValue(T b, CEREAL_RAPIDJSON_ENABLEIF((internal::IsSame))) CEREAL_RAPIDJSON_NOEXCEPT // See #472
+ explicit GenericValue(T b, RAPIDJSON_ENABLEIF((internal::IsSame))) RAPIDJSON_NOEXCEPT // See #472
#else
- explicit GenericValue(bool b) CEREAL_RAPIDJSON_NOEXCEPT
+ explicit GenericValue(bool b) RAPIDJSON_NOEXCEPT
#endif
: data_() {
// safe-guard against failing SFINAE
- CEREAL_RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value));
+ RAPIDJSON_STATIC_ASSERT((internal::IsSame::Value));
data_.f.flags = b ? kTrueFlag : kFalseFlag;
}
//! Constructor for int value.
- explicit GenericValue(int i) CEREAL_RAPIDJSON_NOEXCEPT : data_() {
+ explicit GenericValue(int i) RAPIDJSON_NOEXCEPT : data_() {
data_.n.i64 = i;
data_.f.flags = (i >= 0) ? (kNumberIntFlag | kUintFlag | kUint64Flag) : kNumberIntFlag;
}
//! Constructor for unsigned value.
- explicit GenericValue(unsigned u) CEREAL_RAPIDJSON_NOEXCEPT : data_() {
+ explicit GenericValue(unsigned u) RAPIDJSON_NOEXCEPT : data_() {
data_.n.u64 = u;
data_.f.flags = (u & 0x80000000) ? kNumberUintFlag : (kNumberUintFlag | kIntFlag | kInt64Flag);
}
//! Constructor for int64_t value.
- explicit GenericValue(int64_t i64) CEREAL_RAPIDJSON_NOEXCEPT : data_() {
+ explicit GenericValue(int64_t i64) RAPIDJSON_NOEXCEPT : data_() {
data_.n.i64 = i64;
data_.f.flags = kNumberInt64Flag;
if (i64 >= 0) {
data_.f.flags |= kNumberUint64Flag;
- if (!(static_cast(i64) & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000)))
+ if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000)))
data_.f.flags |= kUintFlag;
- if (!(static_cast(i64) & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
+ if (!(static_cast(i64) & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
data_.f.flags |= kIntFlag;
}
- else if (i64 >= static_cast(CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
+ else if (i64 >= static_cast(RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
data_.f.flags |= kIntFlag;
}
//! Constructor for uint64_t value.
- explicit GenericValue(uint64_t u64) CEREAL_RAPIDJSON_NOEXCEPT : data_() {
+ explicit GenericValue(uint64_t u64) RAPIDJSON_NOEXCEPT : data_() {
data_.n.u64 = u64;
data_.f.flags = kNumberUint64Flag;
- if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0x80000000, 0x00000000)))
+ if (!(u64 & RAPIDJSON_UINT64_C2(0x80000000, 0x00000000)))
data_.f.flags |= kInt64Flag;
- if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000)))
+ if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x00000000)))
data_.f.flags |= kUintFlag;
- if (!(u64 & CEREAL_RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
+ if (!(u64 & RAPIDJSON_UINT64_C2(0xFFFFFFFF, 0x80000000)))
data_.f.flags |= kIntFlag;
}
//! Constructor for double value.
- explicit GenericValue(double d) CEREAL_RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; }
+ explicit GenericValue(double d) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = d; data_.f.flags = kNumberDoubleFlag; }
+
+ //! Constructor for float value.
+ explicit GenericValue(float f) RAPIDJSON_NOEXCEPT : data_() { data_.n.d = static_cast(f); data_.f.flags = kNumberDoubleFlag; }
//! Constructor for constant string (i.e. do not make a copy of string)
- GenericValue(const Ch* s, SizeType length) CEREAL_RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); }
+ GenericValue(const Ch* s, SizeType length) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(StringRef(s, length)); }
//! Constructor for constant string (i.e. do not make a copy of string)
- explicit GenericValue(StringRefType s) CEREAL_RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); }
+ explicit GenericValue(StringRefType s) RAPIDJSON_NOEXCEPT : data_() { SetStringRaw(s); }
//! Constructor for copy-string (i.e. do make a copy of string)
GenericValue(const Ch* s, SizeType length, Allocator& allocator) : data_() { SetStringRaw(StringRef(s, length), allocator); }
@@ -684,9 +845,9 @@ public:
//! Constructor for copy-string (i.e. do make a copy of string)
GenericValue(const Ch*s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); }
-#if CEREAL_RAPIDJSON_HAS_STDSTRING
+#if RAPIDJSON_HAS_STDSTRING
//! Constructor for copy-string from a string object (i.e. do make a copy of string)
- /*! \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING.
+ /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING.
*/
GenericValue(const std::basic_string& s, Allocator& allocator) : data_() { SetStringRaw(StringRef(s), allocator); }
#endif
@@ -697,7 +858,7 @@ public:
\note \c Array is always pass-by-value.
\note the source array is moved into this value and the sourec array becomes empty.
*/
- GenericValue(Array a) CEREAL_RAPIDJSON_NOEXCEPT : data_(a.value_.data_) {
+ GenericValue(Array a) RAPIDJSON_NOEXCEPT : data_(a.value_.data_) {
a.value_.data_ = Data();
a.value_.data_.f.flags = kArrayFlag;
}
@@ -708,7 +869,7 @@ public:
\note \c Object is always pass-by-value.
\note the source object is moved into this value and the sourec object becomes empty.
*/
- GenericValue(Object o) CEREAL_RAPIDJSON_NOEXCEPT : data_(o.value_.data_) {
+ GenericValue(Object o) RAPIDJSON_NOEXCEPT : data_(o.value_.data_) {
o.value_.data_ = Data();
o.value_.data_.f.flags = kObjectFlag;
}
@@ -717,25 +878,30 @@ public:
/*! Need to destruct elements of array, members of object, or copy-string.
*/
~GenericValue() {
- if (Allocator::kNeedFree) { // Shortcut by Allocator's trait
+ // With RAPIDJSON_USE_MEMBERSMAP, the maps need to be destroyed to release
+ // their Allocator if it's refcounted (e.g. MemoryPoolAllocator).
+ if (Allocator::kNeedFree || (RAPIDJSON_USE_MEMBERSMAP+0 &&
+ internal::IsRefCounted::Value)) {
switch(data_.f.flags) {
case kArrayFlag:
{
GenericValue* e = GetElementsPointer();
for (GenericValue* v = e; v != e + data_.a.size; ++v)
v->~GenericValue();
- Allocator::Free(e);
+ if (Allocator::kNeedFree) { // Shortcut by Allocator's trait
+ Allocator::Free(e);
+ }
}
break;
case kObjectFlag:
- for (MemberIterator m = MemberBegin(); m != MemberEnd(); ++m)
- m->~Member();
- Allocator::Free(GetMembersPointer());
+ DoFreeMembers();
break;
case kCopyStringFlag:
- Allocator::Free(const_cast(GetStringPointer()));
+ if (Allocator::kNeedFree) { // Shortcut by Allocator's trait
+ Allocator::Free(const_cast(GetStringPointer()));
+ }
break;
default:
@@ -752,16 +918,22 @@ public:
//! Assignment with move semantics.
/*! \param rhs Source of the assignment. It will become a null value after assignment.
*/
- GenericValue& operator=(GenericValue& rhs) CEREAL_RAPIDJSON_NOEXCEPT {
- CEREAL_RAPIDJSON_ASSERT(this != &rhs);
- this->~GenericValue();
- RawAssign(rhs);
+ GenericValue& operator=(GenericValue& rhs) RAPIDJSON_NOEXCEPT {
+ if (RAPIDJSON_LIKELY(this != &rhs)) {
+ // Can't destroy "this" before assigning "rhs", otherwise "rhs"
+ // could be used after free if it's an sub-Value of "this",
+ // hence the temporary danse.
+ GenericValue temp;
+ temp.RawAssign(rhs);
+ this->~GenericValue();
+ RawAssign(temp);
+ }
return *this;
}
-#if CEREAL_RAPIDJSON_HAS_CXX11_RVALUE_REFS
+#if RAPIDJSON_HAS_CXX11_RVALUE_REFS
//! Move assignment in C++11
- GenericValue& operator=(GenericValue&& rhs) CEREAL_RAPIDJSON_NOEXCEPT {
+ GenericValue& operator=(GenericValue&& rhs) RAPIDJSON_NOEXCEPT {
return *this = rhs.Move();
}
#endif
@@ -771,7 +943,7 @@ public:
\note This overload is needed to avoid clashes with the generic primitive type assignment overload below.
\see GenericStringRef, operator=(T)
*/
- GenericValue& operator=(StringRefType str) CEREAL_RAPIDJSON_NOEXCEPT {
+ GenericValue& operator=(StringRefType str) RAPIDJSON_NOEXCEPT {
GenericValue s(str);
return *this = s;
}
@@ -789,7 +961,7 @@ public:
use \ref SetBool() instead.
*/
template
- CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&))
+ RAPIDJSON_DISABLEIF_RETURN((internal::IsPointer), (GenericValue&))
operator=(T value) {
GenericValue v(value);
return *this = v;
@@ -800,12 +972,13 @@ public:
\tparam SourceAllocator Allocator type of \c rhs
\param rhs Value to copy from (read-only)
\param allocator Allocator to use for copying
+ \param copyConstStrings Force copying of constant strings (e.g. referencing an in-situ buffer)
*/
template
- GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator) {
- CEREAL_RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs));
+ GenericValue& CopyFrom(const GenericValue& rhs, Allocator& allocator, bool copyConstStrings = false) {
+ RAPIDJSON_ASSERT(static_cast(this) != static_cast(&rhs));
this->~GenericValue();
- new (this) GenericValue(rhs, allocator);
+ new (this) GenericValue(rhs, allocator, copyConstStrings);
return *this;
}
@@ -814,7 +987,7 @@ public:
\param other Another value.
\note Constant complexity.
*/
- GenericValue& Swap(GenericValue& other) CEREAL_RAPIDJSON_NOEXCEPT {
+ GenericValue& Swap(GenericValue& other) RAPIDJSON_NOEXCEPT {
GenericValue temp;
temp.RawAssign(*this);
RawAssign(other);
@@ -834,11 +1007,11 @@ public:
\endcode
\see Swap()
*/
- friend inline void swap(GenericValue& a, GenericValue& b) CEREAL_RAPIDJSON_NOEXCEPT { a.Swap(b); }
+ friend inline void swap(GenericValue& a, GenericValue& b) RAPIDJSON_NOEXCEPT { a.Swap(b); }
//! Prepare Value for move semantics
/*! \return *this */
- GenericValue& Move() CEREAL_RAPIDJSON_NOEXCEPT { return *this; }
+ GenericValue& Move() RAPIDJSON_NOEXCEPT { return *this; }
//@}
//!@name Equal-to and not-equal-to operators
@@ -846,7 +1019,7 @@ public:
//! Equal-to operator
/*!
\note If an object contains duplicated named member, comparing equality with any object is always \c false.
- \note Linear time complexity (number of all values in the subtree and total lengths of all strings).
+ \note Complexity is quadratic in Object's member number and linear for the rest (number of all values in the subtree and total lengths of all strings).
*/
template
bool operator==(const GenericValue& rhs) const {
@@ -893,9 +1066,9 @@ public:
//! Equal-to operator with const C-string pointer
bool operator==(const Ch* rhs) const { return *this == GenericValue(StringRef(rhs)); }
-#if CEREAL_RAPIDJSON_HAS_STDSTRING
+#if RAPIDJSON_HAS_STDSTRING
//! Equal-to operator with string object
- /*! \note Requires the definition of the preprocessor symbol \ref CEREAL_RAPIDJSON_HAS_STDSTRING.
+ /*! \note Requires the definition of the preprocessor symbol \ref RAPIDJSON_HAS_STDSTRING.
*/
bool operator==(const std::basic_string& rhs) const { return *this == GenericValue(StringRef(rhs)); }
#endif
@@ -903,7 +1076,7 @@ public:
//! Equal-to operator with primitive types
/*! \tparam T Either \ref Type, \c int, \c unsigned, \c int64_t, \c uint64_t, \c double, \c true, \c false
*/
- template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); }
+ template RAPIDJSON_DISABLEIF_RETURN((internal::OrExpr,internal::IsGenericValue >), (bool)) operator==(const T& rhs) const { return *this == GenericValue(rhs); }
//! Not-equal-to operator
/*! \return !(*this == rhs)
@@ -917,18 +1090,20 @@ public:
//! Not-equal-to operator with arbitrary types
/*! \return !(*this == rhs)
*/
- template CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); }
+ template RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& rhs) const { return !(*this == rhs); }
+#ifndef __cpp_impl_three_way_comparison
//! Equal-to operator with arbitrary types (symmetric version)
/*! \return (rhs == lhs)
*/
- template friend CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; }
+ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator==(const T& lhs, const GenericValue& rhs) { return rhs == lhs; }
//! Not-Equal-to operator with arbitrary types (symmetric version)
/*! \return !(rhs == lhs)
*/
- template friend CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); }
+ template friend RAPIDJSON_DISABLEIF_RETURN((internal::IsGenericValue), (bool)) operator!=(const T& lhs, const GenericValue& rhs) { return !(rhs == lhs); }
//@}
+#endif
//!@name Type
//@{
@@ -955,14 +1130,14 @@ public:
uint64_t u = GetUint64();
volatile double d = static_cast(u);
return (d >= 0.0)
- && (d < static_cast(std::numeric_limits::max()))
+ && (d < static_cast((std::numeric_limits::max)()))
&& (u == static_cast(d));
}
if (IsInt64()) {
int64_t i = GetInt64();
volatile double d = static_cast(i);
- return (d >= static_cast(std::numeric_limits::min()))
- && (d < static_cast(std::numeric_limits::max()))
+ return (d >= static_cast((std::numeric_limits::min)()))
+ && (d < static_cast((std::numeric_limits::max)()))
&& (i == static_cast(d));
}
return true; // double, int, uint are always lossless
@@ -979,8 +1154,8 @@ public:
bool IsLosslessFloat() const {
if (!IsNumber()) return false;
double a = GetDouble();
- if (a < static_cast(-std::numeric_limits::max())
- || a > static_cast(std::numeric_limits::max()))
+ if (a < static_cast(-(std::numeric_limits::max)())
+ || a > static_cast((std::numeric_limits::max)()))
return false;
double b = static_cast(static_cast(a));
return a >= b && a <= b; // Prevent -Wfloat-equal
@@ -998,7 +1173,7 @@ public:
//!@name Bool
//@{
- bool GetBool() const { CEREAL_RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; }
+ bool GetBool() const { RAPIDJSON_ASSERT(IsBool()); return data_.f.flags == kTrueFlag; }
//!< Set boolean value
/*! \post IsBool() == true */
GenericValue& SetBool(bool b) { this->~GenericValue(); new (this) GenericValue(b); return *this; }
@@ -1013,10 +1188,13 @@ public:
GenericValue& SetObject() { this->~GenericValue(); new (this) GenericValue(kObjectType); return *this; }
//! Get the number of members in the object.
- SizeType MemberCount() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return data_.o.size; }
+ SizeType MemberCount() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size; }
+
+ //! Get the capacity of object.
+ SizeType MemberCapacity() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.capacity; }
//! Check whether the object is empty.
- bool ObjectEmpty() const { CEREAL_RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; }
+ bool ObjectEmpty() const { RAPIDJSON_ASSERT(IsObject()); return data_.o.size == 0; }
//! Get a value from an object associated with the name.
/*! \pre IsObject() == true
@@ -1028,12 +1206,12 @@ public:
\note Linear time complexity.
*/
template
- CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) {
+ RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(GenericValue&)) operator[](T* name) {
GenericValue n(StringRef(name));
return (*this)[n];
}
template
- CEREAL_RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; }
+ RAPIDJSON_DISABLEIF_RETURN((internal::NotExpr::Type, Ch> >),(const GenericValue&)) operator[](T* name) const { return const_cast(*this)[name]; }
//! Get a value from an object associated with the name.
/*! \pre IsObject() == true
@@ -1050,21 +1228,36 @@ public:
if (member != MemberEnd())
return member->value;
else {
- CEREAL_RAPIDJSON_ASSERT(false); // see above note
+ RAPIDJSON_ASSERT(false); // see above note
- // This will generate -Wexit-time-destructors in clang
- // static GenericValue NullValue;
- // return NullValue;
-
- // Use static buffer and placement-new to prevent destruction
- static char buffer[sizeof(GenericValue)];
+#if RAPIDJSON_HAS_CXX11
+ // Use thread-local storage to prevent races between threads.
+ // Use static buffer and placement-new to prevent destruction, with
+ // alignas() to ensure proper alignment.
+ alignas(GenericValue) thread_local static char buffer[sizeof(GenericValue)];
return *new (buffer) GenericValue();
+#elif defined(_MSC_VER) && _MSC_VER < 1900
+ // There's no way to solve both thread locality and proper alignment
+ // simultaneously.
+ __declspec(thread) static char buffer[sizeof(GenericValue)];
+ return *new (buffer) GenericValue();
+#elif defined(__GNUC__) || defined(__clang__)
+ // This will generate -Wexit-time-destructors in clang, but that's
+ // better than having under-alignment.
+ __thread static GenericValue buffer;
+ return buffer;
+#else
+ // Don't know what compiler this is, so don't know how to ensure
+ // thread-locality.
+ static GenericValue buffer;
+ return buffer;
+#endif
}
}
template
const GenericValue& operator[](const GenericValue 
-