Support local managment for embedded agent on nginx

external/yq/scripts/acceptance.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+set -e
+
+for test in acceptance_tests/*.sh; do
+  echo "--------------------------------------------------------------"
+  echo "$test"
+  echo "--------------------------------------------------------------"
+  (exec "$test");
+done
+

external/yq/scripts/check.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+if command -v golangci-lint &> /dev/null
+then
+    golangci-lint run --verbose
+else
+  ./bin/golangci-lint run --verbose
+fi
+

external/yq/scripts/copy-docs.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+cp how-it-works.md ../yq-gitbook/.
+cp pkg/yqlib/doc/operators/*.md ../yq-gitbook/operators/.
+cp pkg/yqlib/doc/usage/*.md ../yq-gitbook/usage/.

external/yq/scripts/coverage.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+
+go test -coverprofile=coverage.out -v $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')
+go tool cover -html=coverage.out -o coverage.html

external/yq/scripts/devtools.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+set -ex
+go mod download golang.org/x/tools@latest
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
+wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.13.1

external/yq/scripts/extract-checksum.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# This script works with checksums_hashes_order and checksums to extract the relevant
+# sha of the various yq downloads. You can then use your favourite checksum tool to validate.
+# <CHECKSUM> must match an entry in checksums_hashes_order.
+#
+# Usage: ./extract-checksum.sh <CHECKSUM> <FILE>
+# E.g: ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz
+# Outputs: 
+# yq_linux_amd64.tar.gz	acebc9d07aa2d0e482969b2c080ee306e8f58efbd6f2d857eefbce6469da1473
+#
+# Usage with rhash:
+# ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz | rhash -c -
+#
+# Tip, if you want the checksum first then the filename  (e.g. for the md5sum command)
+# then you can pipe the output of this script into awk to switch the fields around:
+#
+# ./extract-checksum.sh MD5 yq_linux_amd64.tar.gz | awk '{ print $2 " " $1}' | md5sum -c -
+#
+#
+
+if [ "$1" == "" ]; then
+  echo "Please specify at a hash algorithm from the checksum_hashes_order"
+  echo "Usage: $0 <HASH-ALG> <FILE>"
+  exit 1
+fi
+
+if [ "$2" != "" ]; then
+  # so we dont match x.tar.gz when 'x' is given
+  file="$2\s"
+else 
+  file=""
+fi
+
+if [ ! -f "checksums_hashes_order" ]; then
+  echo "This script requires checksums_hashes_order to run"
+  echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate"
+  exit 1
+fi
+
+if [ ! -f "checksums" ]; then
+  echo "This script requires the checksums file to run"
+  echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate"
+  exit 1
+fi
+
+
+grepMatch=$(grep -m 1 -n "$1" checksums_hashes_order)
+if [ "$grepMatch" == "" ]; then
+  echo "Could not find hash algorith '$1' in checksums_hashes_order"
+  exit 1
+fi
+
+set -e
+
+lineNumber=$(echo "$grepMatch" | cut -f1 -d:)
+
+realLineNumber="$(($lineNumber + 1))"
+
+grep "$file" checksums | sed 's/  /\t/g' | cut -f1,$realLineNumber

external/yq/scripts/format.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+gofmt -w -s .
+go mod tidy
+go mod vendor

external/yq/scripts/generate-man-page-md.sh

@@ -0,0 +1,16 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md
+printf "\n# HOW IT WORKS\n" >> man.md
+tail -n +2 how-it-works.md >> man.md
+
+for f in ./pkg/yqlib/doc/operators/*.md; do
+  cat "$f" >> man.md
+done
+
+for f in ./pkg/yqlib/doc/usage/*.md; do
+  cat "$f" >> man.md
+done

external/yq/scripts/generate-man-page.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+pandoc \
+  --variable=title:"YQ" \
+  --variable=section:"1" \
+  --variable=author:"Mike Farah" \
+  --variable=header:"${MAN_HEADER}" \
+  --standalone --to man man.md -o yq.1

external/yq/scripts/install-man-page.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+my_path="$(command -v yq)"
+
+if [ -z "$my_path" ]; then
+  echo "'yq' wasn't found in your PATH, so we don't know where to put the man pages."
+  echo "Please update your PATH to include yq, and run this script again."
+  exit 1
+fi
+
+# ex: ~/.local/bin/yq => ~/.local/
+my_prefix="$(dirname "$(dirname "$(command -v yq)")")"
+mkdir -p "$my_prefix/share/man/man1/"
+cp yq.1 "$my_prefix/share/man/man1/"

external/yq/scripts/release-deb.sh

@@ -0,0 +1,264 @@
+#!/bin/bash -eux
+#
+# Copyright (C) 2021 Roberto Mier Escandón <rmescandon@gmail.com>
+#
+# This script creates a .deb package file with yq valid for ubuntu 20.04 by default
+# You can pass 
+
+DOCKER_IMAGE_NAME=yq-deb-builder
+DOCKER_IMAGE_TAG=$(git describe --always --tags)
+OUTPUT=
+GOVERSION="1.17.4"
+KEYID=
+MAINTAINER=
+DO_PUBLISH=
+PPA="rmescandon/yq"
+VERSION=
+DISTRIBUTION=
+DO_SIGN=
+PASSPHRASE=
+
+show_help() {
+  echo "  usage: $(basename "$0") VERSION [options...]"
+  echo ""
+  echo "  positional arguments"
+  echo "    VERSION"
+  echo ""
+  echo "  optional arguments:"
+  echo "    -h, --help		              Shows this help"
+  echo "    -d, --distribution DISTRO   The distribution to use for the changelog generation. If not provided, last changelog entry"
+  echo "                                  distribution is considered"
+  echo "    --goversion VERSION         The version of golang to use. Default to $GOVERSION"
+  echo "    -k, --sign-key KEYID        Sign the package sources with the provided gpg key id (long format). When not provided this"
+  echo "                                  paramater, the generated sources are not signed"
+  echo "    -s, --sign                  Sign the package sources with a gpg key of the maintainer"
+  echo "    -m, --maintainer WHO        The maintainer used as author of the changelog. git.name and git.email (see git config) is"
+  echo "                                  the considered format"
+  echo "    -o DIR, --output DIR        The path where leaving the generated debian package. Default to a temporary folder if not set"
+  echo "    -p                          The resultant file is being published to ppa"
+  echo "    --ppa PPA                   Push resultant files to indicated ppa. This option should be given along with a signing key."
+  echo "                                  Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'"
+  echo "    --passphrase PASSPHRASE     Passphrase to decrypt the signage key"
+  exit 1
+}
+# read input args
+while [ $# -ne 0 ]; do
+  case $1 in
+    -h|--help)
+      show_help
+      ;;
+    -d|--distribution)
+      shift
+      DISTRIBUTION="$1"
+      ;;
+    --goversion)
+      shift
+      GOVERSION="$1"
+      ;;
+    -k|--sign-key)
+      shift
+      DO_SIGN='y'
+      KEYID="$1"
+      ;;
+    -s|--sign)
+      DO_SIGN='y'
+      ;;
+    -m|--maintainer)
+      shift
+      MAINTAINER="$1"
+      ;;
+    -o|--output)
+      shift
+      OUTPUT="$1"
+      ;;
+    -p)
+      DO_PUBLISH="y"
+      ;;
+    --ppa)
+      shift
+      DO_PUBLISH="y"
+      PPA="$1"
+      ;;
+    --passphrase)
+      shift
+      PASSPHRASE="$1"
+      ;;
+    *)
+      if [ -z "$VERSION" ]; then
+        VERSION="$1"
+      else
+        show_help
+      fi
+  esac
+  shift
+done
+
+[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help)
+
+if [ -n "$OUTPUT" ]; then
+  OUTPUT="$(realpath "$OUTPUT")"
+  mkdir -p "$OUTPUT"
+else
+  # Temporary folder where leaving the built deb package in case that output folder is not provided
+  OUTPUT="$(mktemp -d)"
+fi 
+
+# Define the folders with the source project and the build artifacts and files
+srcdir="$(realpath "$(dirname "$0")"/..)"
+blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")"
+# clean on exit
+cleanup() {
+  rm -f "${blddir}/build.sh" || true
+  rm -f "${blddir}/Dockerfile" || true
+  rm -f "${blddir}/dput.cf" || true
+  docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true
+}
+trap cleanup EXIT INT
+
+# configure the dput config in case publishing is requested
+lp_id="$(echo "$PPA" | cut -d'/' -f1)"
+ppa_id="$(echo "$PPA" | cut -d'/' -f2)"
+cat << EOF > ${blddir}/dput.cf
+[ppa]
+fqdn                    = ppa.launchpad.net
+method                  = ftp
+incoming                = ~${lp_id}/ubuntu/${ppa_id}
+login                   = anonymous
+EOF
+
+# create the main script
+cat << EOF > ${blddir}/build.sh
+#!/bin/bash 
+set -e -o pipefail
+
+PATH=$PATH:/usr/local/go/bin
+export GPG_TTY=$(tty)
+
+go mod vendor
+
+### bump debian/changelog
+
+# maintainer
+export DEBEMAIL="$MAINTAINER"
+if [ -z "$MAINTAINER" ]; then
+  export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)"
+fi
+
+# prepend a 'v' char to complete the tag name from where calculating the changelog
+SINCE="v\$(dpkg-parsechangelog -S Version)"
+
+# distribution
+DISTRIBUTION="$DISTRIBUTION"
+if [ -z "$DISTRIBUTION" ]; then
+  DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)"
+fi
+
+# generate changelog
+gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION"
+
+# using -d to prevent failing when searching for golang dep on control file
+params=("-d" "-S")
+
+# add the -sa option for signing along with the key to use when provided key id
+if [ -n "$DO_SIGN" ]; then
+  params+=("-sa")
+
+  # read from gpg the key id associated with the maintainer if not provided explicitly
+  if [ -z "$KEYID" ]; then
+    KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)"
+  else
+    KEYID="$KEYID"
+  fi
+  
+  params+=("--sign-key="\$KEYID"")
+
+  if [ -n "$PASSPHRASE" ]; then
+    gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000
+    KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)"
+    /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP"
+  fi
+
+else
+  params+=("-us" "-uc")
+fi
+
+debuild  \${params[@]}
+mv ../yq_* /home/yq/output
+
+echo ""
+echo -e "\tfind resulting package at: "$OUTPUT""
+
+# publish to ppa whether given
+if [ -n "$DO_PUBLISH" ]; then
+  dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes
+fi
+EOF
+chmod +x "${blddir}"/build.sh
+
+# build the docker image with all dependencies
+cat << EOF > ${blddir}/Dockerfile
+FROM bitnami/minideb:bullseye as base
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+ENV GO111MODULE on
+ENV GOMODCACHE /home/yq/go
+
+RUN set -e \
+  && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \
+  && apt-get -qq update
+
+# install golang on its $GOVERSION
+FROM base as golang
+RUN apt-get -qq -y --no-install-recommends install \
+    ca-certificates \
+    wget
+RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4
+RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz"
+
+FROM base
+RUN apt-get -qq -y --no-install-recommends install \
+    build-essential \
+    debhelper \
+    devscripts \
+    dput \
+    fakeroot \
+    git-buildpackage \
+    gpg-agent \
+    libdistro-info-perl \
+    pandoc \
+    rsync \
+    sensible-utils && \
+  apt-get clean && \
+  rm -rf /tmp/* /var/tmp/*
+
+COPY --from=golang /usr/local/go /usr/local/go
+
+# build debian package as yq user
+RUN useradd -ms /bin/bash yq && \
+  mkdir /home/yq/src && chown -R yq: /home/yq/src && \
+  mkdir /home/yq/output && chown -R yq: /home/yq/output
+
+ADD ./build/dput.cf /etc/dput.cf
+ADD ./build/build.sh /usr/bin/build.sh
+RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh
+
+USER yq
+
+WORKDIR /home/yq/src
+VOLUME ["/home/yq/src"]
+
+# dir where output packages are finally left
+VOLUME ["/home/yq/output"]
+
+CMD ["/usr/bin/build.sh"]
+EOF
+
+DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" .
+
+docker run --rm -i \
+  -v "${srcdir}":/home/yq/src:delegated \
+  -v "${OUTPUT}":/home/yq/output \
+  -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \
+  -u "$(id -u)" \
+  "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"

external/yq/scripts/secure.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -o errexit
+set -o pipefail
+
+if command -v gosec &> /dev/null
+then
+    gosec "${PWD}" ./...
+else
+  ./bin/gosec "${PWD}" ./...
+fi

external/yq/scripts/setup.sh

@@ -0,0 +1,92 @@
+#!/bin/bash
+
+set -eu
+
+find_mgr() {
+    if hash minishift 2>/dev/null; then
+        echo "minishift"
+    else
+        if hash docker-machine 2>/dev/null; then
+            echo "docker-machine"
+        fi
+    fi
+}
+
+get_vm_name() {
+    case "$1" in
+        minishift)
+            echo "minishift"
+            ;;
+        docker-machine)
+            echo "${DOCKER_MACHINE_NAME}"
+            ;;
+        *)
+            ;;
+    esac
+}
+
+is_vm_running() {
+    local vm=$1
+    declare -a running=($(VBoxManage list runningvms | awk '{ print $1 }'))
+    local result='false'
+
+    for rvm in "${running[@]}"; do
+        if [[ "${rvm}" == *"${vm}"* ]]; then
+            result='true'
+        fi
+    done
+    echo "$result"
+}
+
+if hash cygpath 2>/dev/null; then
+    PROJECT_DIR=$(cygpath -w -a "$(pwd)")
+else
+    PROJECT_DIR=$(pwd)
+fi
+
+VM_MGR=$(find_mgr)
+if [[ -z $VM_MGR ]]; then
+    echo "ERROR: No VM Manager found; expected one of ['minishift', 'docker-machine']"
+    exit 1
+fi
+
+VM_NAME=$(get_vm_name "$VM_MGR")
+if [[ -z $VM_NAME ]]; then
+    echo "ERROR: No VM found; try running 'eval $(docker-machine env)'"
+    exit 1
+fi
+
+if ! hash VBoxManage 2>/dev/null; then
+    echo "VirtualBox executable 'VBoxManage' not found in path"
+    exit 1
+fi
+
+avail=$(is_vm_running "$VM_NAME")
+if [[ "$avail" == *"true"* ]]; then
+    res=$(VBoxManage sharedfolder add "${VM_NAME}" --name "${PROJECT}" --hostpath "${PROJECT_DIR}" --transient 2>&1)
+    if [[ -z $res || $res == *"already exists"* ]]; then
+        # no need to show that it already exists
+        :
+    else
+        echo "$res"
+        exit 1
+    fi
+    echo "VM: [${VM_NAME}] -- Added Sharedfolder [${PROJECT}] @Path [${PROJECT_DIR}]"
+else
+    echo "$VM_NAME is not currently running; please start your VM and try again."
+    exit 1
+fi
+
+SSH_CMD="sudo mkdir -p /${PROJECT} ; sudo mount -t vboxsf ${PROJECT} /${PROJECT}"
+case "${VM_MGR}" in
+    minishift)
+        minishift ssh "${SSH_CMD}"
+        echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]"
+        ;;
+    docker-machine)
+        docker-machine ssh "${VM_NAME}" "${SSH_CMD}"
+        echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]"
+        ;;
+    *)
+        ;;
+esac

external/yq/scripts/test-docker.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+set -e
+
+docker build . -t temp
+docker run --rm -it --entrypoint sh temp -c 'touch a'

external/yq/scripts/test.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')

external/yq/scripts/xcompile.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+# you may need to go install github.com/mitchellh/gox@v1.0.1 first
+
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 darwin/arm64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
+
+cd build
+
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh
+tar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh
+
+rm yq_windows_386.exe.tar.gz
+rm yq_windows_amd64.exe.tar.gz
+
+zip yq_windows_386.zip yq_windows_386.exe
+zip yq_windows_amd64.zip yq_windows_amd64.exe
+
+rm yq.1
+
+rhash -r -a . -o checksums
+
+rhash --list-hashes > checksums_hashes_order
+
+cp ../scripts/extract-checksum.sh .