diff --git a/README.md b/README.md index f7fd96d..94bfe22 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ open-appsec GitHub includes four main repositories: ## Installing external dependencies -Before compiling the services, you'll need to ensure the latest development versions of the following libraries: +Before compiling the services, you'll need to ensure the latest development versions of the following libraries and tools: * Boost * OpenSSL * PCRE2 @@ -107,12 +107,13 @@ Before compiling the services, you'll need to ensure the latest development vers * Redis * Hiredis * MaxmindDB +* yq An example of installing the packages on Alpine: ```bash $ apk update - $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis libmaxminddb-dev + $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis libmaxminddb-dev yq ``` ## Compiling and packaging the agent code diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 1cd077d..172cc05 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,3 +1,2 @@ add_subdirectory(graphqlparser) add_subdirectory(yajl) -add_subdirectory(yq) diff --git a/external/yq/.dockerignore b/external/yq/.dockerignore deleted file mode 100755 index 36f971e..0000000 --- a/external/yq/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -bin/* diff --git a/external/yq/.golangci.yml b/external/yq/.golangci.yml deleted file mode 100755 index 84642b6..0000000 --- a/external/yq/.golangci.yml +++ /dev/null @@ -1,40 +0,0 @@ -run: - timeout: 5m -linters: - enable: - - asciicheck - - depguard - - errorlint - - gci - - gochecknoinits - - gofmt - - goimports - - gosec - - megacheck - - misspell - - nakedret - - nolintlint - - predeclared - - revive - - unconvert - - unparam -linters-settings: - depguard: - list-type: blacklist - include-go-root: true - packages: - - io/ioutil - packages-with-error-message: - - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead." -issues: - exclude-rules: - - linters: - - gosec - text: "Implicit memory aliasing in for loop." - path: _test\.go - - linters: - - revive - text: "unexported-return" - - linters: - - revive - text: "var-naming" diff --git a/external/yq/CMakeLists.txt b/external/yq/CMakeLists.txt deleted file mode 100755 index 9e55586..0000000 --- a/external/yq/CMakeLists.txt +++ /dev/null @@ -1 +0,0 @@ -install(FILES yq_linux_amd64 DESTINATION orchestration/scripts RENAME yq PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) diff --git a/external/yq/CODE_OF_CONDUCT.md b/external/yq/CODE_OF_CONDUCT.md deleted file mode 100755 index d2a542d..0000000 --- a/external/yq/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,76 +0,0 @@ -# Contributor Covenant Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to making participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies both within project spaces and in public spaces -when an individual is representing the project or its community. Examples of -representing a project or community include using an official project e-mail -address, posting via an official social media account, or acting as an appointed -representative at an online or offline event. Representation of a project may be -further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at mikefarah@gmail.com. All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq diff --git a/external/yq/CONTRIBUTING.md b/external/yq/CONTRIBUTING.md deleted file mode 100755 index aa44043..0000000 --- a/external/yq/CONTRIBUTING.md +++ /dev/null @@ -1,40 +0,0 @@ -# Development - -1. Install (golang)[https://golang.org/] -1. Run `scripts/devtools.sh` to install the required devtools -2. Run `make [local] vendor` to install the vendor dependencies -2. Run `make [local] test` to ensure you can run the existing tests -3. Write unit tests - (see existing examples). Changes will not be accepted without corresponding unit tests. -4. Make the code changes. -5. `make [local] test` to lint code and run tests -6. Profit! ok no profit, but raise a PR and get kudos :) - - -# Documentation - -The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined. - -Documentation is written in markdown, and is published in the 'gitbook' branch. - -The various operator documentation (e.g. 'strings') are generated from the 'master' branch, and have a statically defined header (e.g. `pkg/yqlib/doc/operators/headers/add.md`) and the bulk of the docs are generated from the unit tests e.g. `pkg/yqlib/operator_add_test.go`. - -The pipeline will run the tests and automatically concatenate the files together, and put them under -`pkg/qylib/doc/add.md`. These files are checked in the master branch (and are copied to the gitbook branch as part of the release process). - -## How to contribute - -The first step is to find if what you want is automatically generated or not - start by looking in the master branch. - -### Updating dynamic documentation from master -- Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file -- Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run: -`go test -run TestAddOperatorScenarios` which will run that test defined in the `operator_add_test.go` file. -- Ensure the tests still pass, and check the generated documentation have your update. -- Note: If the documentation is only in a `headers/*.md` file, then just update that directly -- Raise a PR to merge the changes into master! - -### Updating static documentation from the gitbook branch -If you haven't found what you want to update in the master branch, then check the gitbook branch directly as there are a few pages in there that are not in master. - -- Update the `*.md` files -- Raise a PR to merge the changes into gitbook. \ No newline at end of file diff --git a/external/yq/Dockerfile b/external/yq/Dockerfile deleted file mode 100755 index 2037624..0000000 --- a/external/yq/Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:1.19.3 as builder - -WORKDIR /go/src/mikefarah/yq - -COPY . . - -RUN CGO_ENABLED=0 go build . -# RUN ./scripts/test.sh -- this too often times out in the github pipeline. -RUN ./scripts/acceptance.sh - -# Choose alpine as a base image to make this useful for CI, as many -# CI tools expect an interactive shell inside the container -FROM alpine:3 as production -LABEL maintainer="Mike Farah " - -COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq - -WORKDIR /workdir - -RUN set -eux; \ - addgroup -g 1000 yq; \ - adduser -u 1000 -G yq -s /bin/sh -h /home/yq -D yq - -RUN chown -R yq:yq /workdir - -USER yq - -ENTRYPOINT ["/usr/bin/yq"] diff --git a/external/yq/Dockerfile.dev b/external/yq/Dockerfile.dev deleted file mode 100755 index 0fc879f..0000000 --- a/external/yq/Dockerfile.dev +++ /dev/null @@ -1,10 +0,0 @@ -FROM golang:1.19.3 - -COPY scripts/devtools.sh /opt/devtools.sh - -RUN set -e -x \ - && /opt/devtools.sh -ENV PATH=/go/bin:$PATH - -ENV CGO_ENABLED 0 -ENV GOPATH /go:/yq diff --git a/external/yq/LICENSE b/external/yq/LICENSE deleted file mode 100755 index dfbcc19..0000000 --- a/external/yq/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Mike Farah - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/external/yq/Makefile.variables b/external/yq/Makefile.variables deleted file mode 100755 index b1968ba..0000000 --- a/external/yq/Makefile.variables +++ /dev/null @@ -1,38 +0,0 @@ -export PROJECT = yq -IMPORT_PATH := github.com/mikefarah/${PROJECT} - -export GIT_COMMIT = $(shell git rev-parse --short HEAD) -export GIT_DIRTY = $(shell test -n "$$(git status --porcelain)" && echo "+CHANGES" || true) -export GIT_DESCRIBE = $(shell git describe --tags --always) -LDFLAGS := -LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} -LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE} - -GITHUB_TOKEN ?= - -# Windows environment? -CYG_CHECK := $(shell hash cygpath 2>/dev/null && echo 1) -ifeq ($(CYG_CHECK),1) - VBOX_CHECK := $(shell hash VBoxManage 2>/dev/null && echo 1) - - # Docker Toolbox (pre-Windows 10) - ifeq ($(VBOX_CHECK),1) - ROOT := /${PROJECT} - else - # Docker Windows - ROOT := $(shell cygpath -m -a "$(shell pwd)") - endif -else - # all non-windows environments - ROOT := $(shell pwd) -endif - -DEV_IMAGE := ${PROJECT}_dev - -ENGINERUN := ${ENGINE} run --rm \ - -e LDFLAGS="${LDFLAGS}" \ - -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ - -v ${ROOT}/vendor:/go/src \ - -v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH} \ - -w /${PROJECT}/src/${IMPORT_PATH} \ - ${DEV_IMAGE} diff --git a/external/yq/README.md b/external/yq/README.md deleted file mode 100755 index dbf5298..0000000 --- a/external/yq/README.md +++ /dev/null @@ -1,373 +0,0 @@ -# yq - -![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg) - - -a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. - -yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. - -## Quick Usage Guide - -Read a value: -```bash -yq '.a.b[0].c' file.yaml -``` - -Pipe from STDIN: -```bash -yq '.a.b[0].c' < file.yaml -``` - -Update a yaml file, inplace -```bash -yq -i '.a.b[0].c = "cool"' file.yaml -``` - -Update using environment variables -```bash -NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml -``` - -Merge multiple files -```bash -# note the use of `ea` to evaluate all the files at once -# instead of in sequence -yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml -``` - -Multiple updates to a yaml file -```bash -yq -i ' - .a.b[0].c = "cool" | - .x.y.z = "foobar" | - .person.name = strenv(NAME) -' file.yaml -``` - -Convert JSON to YAML -```bash -yq -P sample.json -``` - -See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples. - -Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell) - -## Install - -### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest) - -### wget -Use wget to download, gzipped pre-compiled binaries: - - -For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64 - -#### Compressed via tar.gz -```bash -wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\ - tar xz && mv ${BINARY} /usr/bin/yq -``` - -#### Plain binary - -```bash -wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq &&\ - chmod +x /usr/bin/yq -``` - -#### Latest version - -```bash -wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\ - chmod +x /usr/bin/yq -``` - -### MacOS / Linux via Homebrew: -Using [Homebrew](https://brew.sh/) -``` -brew install yq -``` - -### Linux via snap: -``` -snap install yq -``` - -#### Snap notes -`yq` installs with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can: - -``` -sudo cat /etc/myfile | yq '.a.path' -``` - -And to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge): -``` -sudo cat /etc/myfile | yq '.a.path = "value"' | sudo sponge /etc/myfile -``` -or write to a temporary file: -``` -sudo cat /etc/myfile | yq '.a.path = "value"' | sudo tee /etc/myfile.tmp -sudo mv /etc/myfile.tmp /etc/myfile -rm /etc/myfile.tmp -``` - -### Run with Docker or Podman -#### Oneshot use: - -```bash -docker run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression ]FILE... -``` - -Note that you can run `yq` in docker without network access and other privileges if you desire, -namely `--security-opt=no-new-privileges --cap-drop all --network none`. - -```bash -podman run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression ]FILE... -``` - -#### Pipe in via STDIN: - -You'll need to pass the `-i\--interactive` flag to docker: - -```bash -docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml -``` - -```bash -podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml -``` - -#### Run commands interactively: - -```bash -docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq -``` - -```bash -podman run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq -``` - -It can be useful to have a bash function to avoid typing the whole docker command: - -```bash -yq() { - docker run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@" -} -``` - -```bash -yq() { - podman run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@" -} -``` -#### Running as root: - -`yq`'s container image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the container image, or you're having permissions issues when attempting to read/write files you'll need to either: - - -``` -docker run --user="root" -it --entrypoint sh mikefarah/yq -``` - -``` -podman run --user="root" -it --entrypoint sh mikefarah/yq -``` - -Or, in your Dockerfile: - -``` -FROM mikefarah/yq - -USER root -RUN apk add --no-cache bash -USER yq -``` - -#### Missing timezone data -By default, the alpine image yq uses does not include timezone data. If you'd like to use the `tz` operator, you'll need to include this data: - -``` -FROM mikefarah/yq - -USER root -RUN apk add --no-cache tzdata -USER yq -``` - -#### Podman with SELinux - -If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount: - -``` --v "${PWD}":/workdir:z -``` - -### GitHub Action -``` - - name: Set foobar to cool - uses: mikefarah/yq@master - with: - cmd: yq -i '.foo.bar = "cool"' 'config.yml' - - name: Get an entry with a variable that might contain dots or spaces - id: get_username - uses: mikefarah/yq@master - with: - cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml - - name: Reuse a variable obtained in another step - run: echo ${{ steps.get_username.outputs.result }} -``` - -See https://mikefarah.gitbook.io/yq/usage/github-action for more. - -### Go Install: -``` -go install github.com/mikefarah/yq/v4@latest -``` - -## Community Supported Installation methods -As these are supported by the community :heart: - however, they may be out of date with the officially supported releases. - - -### Nix - -``` -nix profile install nixpkgs#yq-go -``` - -See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go) - - -### Webi - -``` -webi yq -``` - -See [webi](https://webinstall.dev/) -Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq) - -### Arch Linux - -``` -pacman -S go-yq -``` - -### Windows: -[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq) -[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq) -``` -choco install yq -``` -Supported by @chillum (https://chocolatey.org/packages/yq) - -and - -### Winget -winget install yq - -https://winget.run/pkg/MikeFarah/yq - -### Mac: -Using [MacPorts](https://www.macports.org/) -``` -sudo port selfupdate -sudo port install yq -``` -Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot) - -### Alpine Linux -- Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories``` -- Update database index with ```apk update``` -- Install yq with ```apk add yq``` - -Supported by Tuan Hoang -https://pkgs.alpinelinux.org/package/edge/community/x86/yq - - -### On Ubuntu 16.04 or higher from Debian package: -```sh -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64 -sudo add-apt-repository ppa:rmescandon/yq -sudo apt update -sudo apt install yq -y -``` -Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq) - -## Features -- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/) -- Written in portable go, so you can download a lovely dependency free binary -- Uses similar syntax as `jq` but works with YAML, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files -- Fully supports multi document yaml files -- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble) -- Colorized yaml output -- [Date/Time manipulation and formatting with TZ](https://mikefarah.gitbook.io/yq/operators/datetime) -- [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) -- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) -- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators). -- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) -- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map) -- Keeps yaml formatting and comments when updating (though there are issues with whitespace) -- [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode) -- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load) -- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert) -- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml) -- [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) -- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv) -- [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion) -- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things. -- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus) - -## [Usage](https://mikefarah.gitbook.io/yq/) - -Check out the [documentation](https://mikefarah.gitbook.io/yq/) for more detailed and advanced usage. - -``` -Usage: - yq [flags] - yq [command] - -Examples: - -# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples. -yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml" - -yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace - - -Available Commands: - completion Generate the autocompletion script for the specified shell - eval (default) Apply the expression to each document in each yaml file in sequence - eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once - help Help about any command - shell-completion Generate completion script - -Flags: - -C, --colors force print with colors - -e, --exit-status set exit status if there are no matches or null or false is returned - -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact - --header-preprocess Slurp any header comments and separators before processing expression. (default true) - -h, --help help for yq - -I, --indent int sets indent level for output (default 2) - -i, --inplace update the file inplace of first file given. - -p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml") - -M, --no-colors force print with no colors - -N, --no-doc Don't print document separators (---) - -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch. - -o, --output-format string [yaml|y|json|j|props|p|xml|x] output format type. (default "yaml") - -P, --prettyPrint pretty print, shorthand for '... style = ""' - -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. - --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments (default true) - -v, --verbose verbose mode - -V, --version Print version information and quit - --xml-attribute-prefix string prefix for xml attributes (default "+") - --xml-content-name string name for xml content (if no attribute name is present). (default "+content") - -Use "yq [command] --help" for more information about a command. -``` -## Known Issues / Missing Features -- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details) -- Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell) - -See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions. diff --git a/external/yq/acceptance_tests/bad_args.sh b/external/yq/acceptance_tests/bad_args.sh deleted file mode 100755 index 34d43b6..0000000 --- a/external/yq/acceptance_tests/bad_args.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -testWriteInPlacePipeIn() { - result=$(./yq e -i -n '.a' 2>&1) - assertEquals 1 $? - assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result" -} - -testWriteInPlacePipeInEvalall() { - result=$(./yq ea -i -n '.a' 2>&1) - assertEquals 1 $? - assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result" -} - -testWriteInPlaceWithSplit() { - result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1) - assertEquals 1 $? - assertEquals "Error: write inplace cannot be used with split file" "$result" -} - -testWriteInPlaceWithSplitEvalAll() { - result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1) - assertEquals 1 $? - assertEquals "Error: write inplace cannot be used with split file" "$result" -} - -testNullWithFiles() { - result=$(./yq e -n '.a = "thing"' test.yml 2>&1) - assertEquals 1 $? - assertEquals "Error: cannot pass files in when using null-input flag" "$result" -} - -testNullWithFilesEvalAll() { - result=$(./yq ea -n '.a = "thing"' test.yml 2>&1) - assertEquals 1 $? - assertEquals "Error: cannot pass files in when using null-input flag" "$result" -} - - - -source ./scripts/shunit2 \ No newline at end of file diff --git a/external/yq/acceptance_tests/basic.sh b/external/yq/acceptance_tests/basic.sh deleted file mode 100755 index 141122c..0000000 --- a/external/yq/acceptance_tests/basic.sh +++ /dev/null @@ -1,356 +0,0 @@ -#!/bin/bash - -setUp() { - rm test*.yml 2>/dev/null || true - rm .xyz 2>/dev/null || true - rm instructions.txt 2>/dev/null || true -} - -testBasicEvalRoundTrip() { - ./yq -n ".a = 123" > test.yml - X=$(./yq '.a' test.yml) - assertEquals 123 "$X" -} - -testBasicTrailingContent() { - cat >test-trailing.yml <test-trailing.yml <test-trailing.yml <test-trailing.yml < test.yml - X=$(cat test.yml | ./yq '.') - assertEquals "a: 123" "$X" -} - -testBasicExpressionMatchesFileName() { - ./yq -n ".xyz = 123" > test.yml - touch .xyz - - X=$(./yq --expression '.xyz' test.yml) - assertEquals "123" "$X" - - X=$(./yq ea --expression '.xyz' test.yml) - assertEquals "123" "$X" -} - -testBasicExpressionFromFile() { - ./yq -n ".xyz = 123" > test.yml - echo '.xyz = "meow" | .cool = "frog"' > instructions.txt - - X=$(./yq --from-file instructions.txt test.yml -o=j -I=0) - assertEquals '{"xyz":"meow","cool":"frog"}' "$X" - - X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0) - assertEquals '{"xyz":"meow","cool":"frog"}' "$X" -} - -testBasicGitHubAction() { - ./yq -n ".a = 123" > test.yml - X=$(cat /dev/null | ./yq test.yml) - assertEquals "a: 123" "$X" - - X=$(cat /dev/null | ./yq e test.yml) - assertEquals "a: 123" "$X" - - X=$(cat /dev/null | ./yq ea test.yml) - assertEquals "a: 123" "$X" -} - -testBasicGitHubActionWithExpression() { - ./yq -n ".a = 123" > test.yml - X=$(cat /dev/null | ./yq '.a' test.yml) - assertEquals "123" "$X" - - X=$(cat /dev/null | ./yq e '.a' test.yml) - assertEquals "123" "$X" - - X=$(cat /dev/null | ./yq ea '.a' test.yml) - assertEquals "123" "$X" -} - - -testBasicEvalAllAllFiles() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(./yq ea test.yml test2.yml) - Y=$(./yq e '.' test.yml test2.yml) - assertEquals "$Y" "$X" -} - -# when given a file, don't read STDIN -# otherwise strange things start happening -# in scripts -# https://github.com/mikefarah/yq/issues/1115 - -testBasicCatWithFilesNoDash() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq test2.yml) - Y=$(./yq e '.' test2.yml) - assertEquals "$Y" "$X" -} - -# when the nullinput flag is used -# dont automatically read STDIN (this breaks github actions) -testBasicCreateFileGithubAction() { - cat /dev/null | ./yq -n ".a = 123" > test.yml -} - -testBasicEvalAllCatWithFilesNoDash() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq ea test2.yml) - Y=$(./yq e '.' test2.yml) - assertEquals "$Y" "$X" -} - -testBasicCatWithFilesNoDashWithExp() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq '.a' test2.yml) - Y=$(./yq e '.a' test2.yml) - assertEquals "$Y" "$X" -} - -testBasicEvalAllCatWithFilesNoDashWithExp() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq ea '.a' test2.yml) - Y=$(./yq e '.a' test2.yml) - assertEquals "$Y" "$X" -} - - -testBasicStdInWithFiles() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq - test2.yml) - Y=$(./yq e '.' test.yml test2.yml) - assertEquals "$Y" "$X" -} - -testBasicEvalAllStdInWithFiles() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq ea - test2.yml) - Y=$(./yq e '.' test.yml test2.yml) - assertEquals "$Y" "$X" -} - -testBasicStdInWithFilesReverse() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq test2.yml -) - Y=$(./yq e '.' test2.yml test.yml) - assertEquals "$Y" "$X" -} - -testBasicEvalAllStdInWithFilesReverse() { - ./yq -n ".a = 123" > test.yml - ./yq -n ".a = 124" > test2.yml - X=$(cat test.yml | ./yq ea test2.yml -) - Y=$(./yq e '.' test2.yml test.yml) - assertEquals "$Y" "$X" -} - -testBasicEvalRoundTripNoEval() { - ./yq -n ".a = 123" > test.yml - X=$(./yq '.a' test.yml) - assertEquals 123 "$X" -} - -testBasicStdInWithOneArg() { - ./yq e -n ".a = 123" > test.yml - X=$(cat test.yml | ./yq e ".a") - assertEquals 123 "$X" - - X=$(cat test.yml | ./yq ea ".a") - assertEquals 123 "$X" - - X=$(cat test.yml | ./yq ".a") - assertEquals 123 "$X" -} - -testBasicUpdateInPlaceSequence() { - cat >test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml < test.yml - X=$(./yq e '.z' test.yml) - assertEquals "null" "$X" -} - -testBasicExitStatus() { - echo "a: cat" > test.yml - X=$(./yq e -e '.z' test.yml 2&>/dev/null) - assertEquals 1 "$?" -} - -testBasicExitStatusNoEval() { - echo "a: cat" > test.yml - X=$(./yq -e '.z' test.yml 2&>/dev/null) - assertEquals 1 "$?" -} - -testBasicExtractFieldWithSeperator() { - cat >test.yml <test.yml <test.yml <&1) - assertEquals 0 $? - assertContains "$result" "Completion ended with directive:" -} - -source ./scripts/shunit2 diff --git a/external/yq/acceptance_tests/empty.sh b/external/yq/acceptance_tests/empty.sh deleted file mode 100755 index 63d5ca8..0000000 --- a/external/yq/acceptance_tests/empty.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -setUp() { - rm test*.yml || true - cat >test.yml <test.yml - X=$(./yq e test.yml) - expected=$(cat test.yml) - assertEquals 0 $? - assertEquals "$expected" "$X" -} - -testEmptyEvalNoNewLineWithExpression() { - echo -n "# comment" >test.yml - X=$(./yq e '.apple = "tree"' test.yml) - read -r -d '' expected << EOM -# comment -apple: tree -EOM - assertEquals "$expected" "$X" -} - -testEmptyEvalPipe() { - X=$(./yq e - < test.yml) - assertEquals 0 $? -} - -testEmptyCommentsWithExpressionEval() { - read -r -d '' expected << EOM -# comment -apple: tree -EOM - - X=$(./yq e '.apple="tree"' test.yml) - - assertEquals "$expected" "$X" -} - -testEmptyCommentsWithExpressionEvalAll() { - read -r -d '' expected << EOM -# comment -apple: tree -EOM - - X=$(./yq ea '.apple="tree"' test.yml) - - assertEquals "$expected" "$X" -} - -testEmptyWithExpressionEval() { - rm test.yml - touch test.yml - expected="apple: tree" - - X=$(./yq e '.apple="tree"' test.yml) - - assertEquals "$expected" "$X" -} - -testEmptyWithExpressionEvalAll() { - rm test.yml - touch test.yml - expected="apple: tree" - - X=$(./yq ea '.apple="tree"' test.yml) - - assertEquals "$expected" "$X" -} - - -source ./scripts/shunit2 \ No newline at end of file diff --git a/external/yq/acceptance_tests/front-matter.sh b/external/yq/acceptance_tests/front-matter.sh deleted file mode 100755 index a9f079b..0000000 --- a/external/yq/acceptance_tests/front-matter.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -setUp() { - rm test*.yml || true - cat >test.yml <test.yml <test.yml <test.yml <test.yml </dev/null || true - rm test*.properties 2>/dev/null || true - rm test*.csv 2>/dev/null || true - rm test*.tsv 2>/dev/null || true - rm test*.xml 2>/dev/null || true -} - -testInputProperties() { - cat >test.properties <test.properties <test.csv <test.tsv <test.yml <BiBi -EOL - - read -r -d '' expected << EOM -cat: - +content: BiBi - +@legs: "4" -EOM - - X=$(./yq e -p=xml test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea -p=xml test.yml) - assertEquals "$expected" "$X" -} - -testInputXmlNamespaces() { - cat >test.yml < - - -EOL - - read -r -d '' expected << EOM -+p_xml: version="1.0" -map: - +@xmlns: some-namespace - +@xmlns:xsi: some-instance - +@xsi:schemaLocation: some-url -EOM - - X=$(./yq e -p=xml test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea -p=xml test.yml) - assertEquals "$expected" "$X" -} - -testInputXmlRoundtrip() { - cat >test.yml < - -Meow -EOL - - read -r -d '' expected << EOM - - -Meow -EOM - - X=$(./yq -p=xml -o=xml test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea -p=xml -o=xml test.yml) - assertEquals "$expected" "$X" -} - - -testInputXmlStrict() { - cat >test.yml < - - -]> - - &writer;©right; - -EOL - - X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1) - assertEquals 1 $? - assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X" - - X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1) - assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X" -} - -testInputXmlGithubAction() { - cat >test.yml <BiBi -EOL - - read -r -d '' expected << EOM -cat: - +content: BiBi - +@legs: "4" -EOM - - X=$(cat /dev/null | ./yq e -p=xml test.yml) - assertEquals "$expected" "$X" - - X=$(cat /dev/null | ./yq ea -p=xml test.yml) - assertEquals "$expected" "$X" -} - -source ./scripts/shunit2 \ No newline at end of file diff --git a/external/yq/acceptance_tests/leading-seperator.sh b/external/yq/acceptance_tests/leading-seperator.sh deleted file mode 100755 index 71217b9..0000000 --- a/external/yq/acceptance_tests/leading-seperator.sh +++ /dev/null @@ -1,453 +0,0 @@ -#!/bin/bash - - -# examples where header-preprocess is required - -setUp() { - rm test*.yml || true - cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <&1) - assertEquals 1 $? - assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result" -} - -testLoadFileExpNotExist() { - result=$(./yq e -n 'load(.a)' 2>&1) - assertEquals 1 $? - assertEquals "Error: Filename expression returned nil" "$result" -} - -testStrLoadFileNotExist() { - result=$(./yq e -n 'strload("cat.yml")' 2>&1) - assertEquals 1 $? - assertEquals "Error: Failed to load cat.yml: open cat.yml: no such file or directory" "$result" -} - -testStrLoadFileExpNotExist() { - result=$(./yq e -n 'strload(.a)' 2>&1) - assertEquals 1 $? - assertEquals "Error: Filename expression returned nil" "$result" -} - -source ./scripts/shunit2 \ No newline at end of file diff --git a/external/yq/acceptance_tests/output-format.sh b/external/yq/acceptance_tests/output-format.sh deleted file mode 100755 index 268185d..0000000 --- a/external/yq/acceptance_tests/output-format.sh +++ /dev/null @@ -1,274 +0,0 @@ -#!/bin/bash - -setUp() { - rm test*.yml || true -} - -testOutputJsonDeprecated() { - cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml < - - cat - - -EOM - - X=$(./yq e --output-format=xml test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea --output-format=xml test.yml) - assertEquals "$expected" "$X" -} - -testOutputXmlShort() { - cat >test.yml < - - cat - - -EOM - - X=$(./yq e --output-format=x test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea --output-format=x test.yml) - assertEquals "$expected" "$X" -} - -testOutputXmComplex() { - cat >test.yml < - - cat - dog - - -EOM - - X=$(./yq e --output-format=x test.yml) - assertEquals "$expected" "$X" - - X=$(./yq ea --output-format=x test.yml) - assertEquals "$expected" "$X" -} - -source ./scripts/shunit2 \ No newline at end of file diff --git a/external/yq/acceptance_tests/pipe.sh b/external/yq/acceptance_tests/pipe.sh deleted file mode 100755 index 43709de..0000000 --- a/external/yq/acceptance_tests/pipe.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash - -setUp() { - rm test*.yml || true - cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test_splitExp.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <)") - rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. )") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") - rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") - - rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") - rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") - - rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output") - rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit") - rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.") - rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml") - rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true" - - rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'") - rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned") - - rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") - rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") - rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") - rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.") - rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") - - rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.") - rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.") - - rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.") - - rootCmd.AddCommand( - createEvaluateSequenceCommand(), - createEvaluateAllCommand(), - completionCmd, - ) - return rootCmd -} diff --git a/external/yq/cmd/shell-completion.go b/external/yq/cmd/shell-completion.go deleted file mode 100755 index 5431abe..0000000 --- a/external/yq/cmd/shell-completion.go +++ /dev/null @@ -1,61 +0,0 @@ -package cmd - -import ( - "os" - - "github.com/spf13/cobra" -) - -var completionCmd = &cobra.Command{ - Use: "shell-completion [bash|zsh|fish|powershell]", - Short: "Generate completion script", - Long: `To load completions: - -Bash: - -$ source <(yq shell-completion bash) - -# To load completions for each session, execute once: -Linux: - $ yq shell-completion bash > /etc/bash_completion.d/yq -MacOS: - $ yq shell-completion bash > /usr/local/etc/bash_completion.d/yq - -Zsh: - -# If shell completion is not already enabled in your environment you will need -# to enable it. You can execute the following once: - -$ echo "autoload -U compinit; compinit" >> ~/.zshrc - -# To load completions for each session, execute once: -$ yq shell-completion zsh > "${fpath[1]}/_yq" - -# You will need to start a new shell for this setup to take effect. - -Fish: - -$ yq shell-completion fish | source - -# To load completions for each session, execute once: -$ yq shell-completion fish > ~/.config/fish/completions/yq.fish -`, - DisableFlagsInUseLine: true, - ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, - Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), - RunE: func(cmd *cobra.Command, args []string) error { - var err error = nil - switch args[0] { - case "bash": - err = cmd.Root().GenBashCompletion(os.Stdout) - case "zsh": - err = cmd.Root().GenZshCompletion(os.Stdout) - case "fish": - err = cmd.Root().GenFishCompletion(os.Stdout, true) - case "powershell": - err = cmd.Root().GenPowerShellCompletion(os.Stdout) - } - return err - - }, -} diff --git a/external/yq/cmd/unwrap_flag.go b/external/yq/cmd/unwrap_flag.go deleted file mode 100755 index e844f2f..0000000 --- a/external/yq/cmd/unwrap_flag.go +++ /dev/null @@ -1,46 +0,0 @@ -package cmd - -import ( - "strconv" - - "github.com/spf13/pflag" -) - -type boolFlag interface { - pflag.Value - IsExplicitySet() bool - IsSet() bool -} - -type unwrapScalarFlagStrc struct { - explicitySet bool - value bool -} - -func newUnwrapFlag() boolFlag { - return &unwrapScalarFlagStrc{value: true} -} - -func (f *unwrapScalarFlagStrc) IsExplicitySet() bool { - return f.explicitySet -} - -func (f *unwrapScalarFlagStrc) IsSet() bool { - return f.value -} - -func (f *unwrapScalarFlagStrc) String() string { - return strconv.FormatBool(f.value) -} - -func (f *unwrapScalarFlagStrc) Set(value string) error { - - v, err := strconv.ParseBool(value) - f.value = v - f.explicitySet = true - return err -} - -func (*unwrapScalarFlagStrc) Type() string { - return "bool" -} diff --git a/external/yq/cmd/utils.go b/external/yq/cmd/utils.go deleted file mode 100755 index 8dad7c3..0000000 --- a/external/yq/cmd/utils.go +++ /dev/null @@ -1,178 +0,0 @@ -package cmd - -import ( - "fmt" - "io" - "os" - - "github.com/mikefarah/yq/v4/pkg/yqlib" - "github.com/spf13/cobra" - "gopkg.in/op/go-logging.v1" -) - -func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { - cmd.SilenceUsage = true - - fileInfo, _ := os.Stdout.Stat() - - if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { - colorsEnabled = true - } - - expression, args, err := processArgs(args) - if err != nil { - return "", nil, err - } - - if splitFileExpFile != "" { - splitExpressionBytes, err := os.ReadFile(splitFileExpFile) - if err != nil { - return "", nil, err - } - splitFileExp = string(splitExpressionBytes) - } - - // backwards compatibility - if outputToJSON { - outputFormat = "json" - } - - if writeInplace && (len(args) == 0 || args[0] == "-") { - return "", nil, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file") - } - - if frontMatter != "" && len(args) == 0 { - return "", nil, fmt.Errorf("front matter flag only applicable when giving an expression and at least one file") - } - - if writeInplace && splitFileExp != "" { - return "", nil, fmt.Errorf("write inplace cannot be used with split file") - } - - if nullInput && len(args) > 0 { - return "", nil, fmt.Errorf("cannot pass files in when using null-input flag") - } - - return expression, args, nil -} - -func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { - yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat) - if err != nil { - return nil, err - } - switch yqlibInputFormat { - case yqlib.XMLInputFormat: - return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil - case yqlib.PropertiesInputFormat: - return yqlib.NewPropertiesDecoder(), nil - case yqlib.JsonInputFormat: - return yqlib.NewJSONDecoder(), nil - case yqlib.CSVObjectInputFormat: - return yqlib.NewCSVObjectDecoder(','), nil - case yqlib.TSVObjectInputFormat: - return yqlib.NewCSVObjectDecoder('\t'), nil - } - prefs := yqlib.ConfiguredYamlPreferences - prefs.EvaluateTogether = evaluateTogether - return yqlib.NewYamlDecoder(prefs), nil -} - -func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) { - - var printerWriter yqlib.PrinterWriter - - if splitFileExp != "" { - colorsEnabled = forceColor - splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp) - if err != nil { - return nil, fmt.Errorf("bad split document expression: %w", err) - } - printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format) - } else { - printerWriter = yqlib.NewSinglePrinterWriter(out) - } - return printerWriter, nil -} - -func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder { - switch format { - case yqlib.JSONOutputFormat: - return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar) - case yqlib.PropsOutputFormat: - return yqlib.NewPropertiesEncoder(unwrapScalar) - case yqlib.CSVOutputFormat: - return yqlib.NewCsvEncoder(',') - case yqlib.TSVOutputFormat: - return yqlib.NewCsvEncoder('\t') - case yqlib.YamlOutputFormat: - return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences) - case yqlib.XMLOutputFormat: - return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences) - } - panic("invalid encoder") -} - -// this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything) -// and being able to call yq with the filename as a single parameter -// -// without this - yq detects there is stdin (thanks githubactions), -// then tries to parse the filename as an expression -func maybeFile(str string) bool { - yqlib.GetLogger().Debugf("checking '%v' is a file", str) - stat, err := os.Stat(str) // #nosec - result := err == nil && !stat.IsDir() - if yqlib.GetLogger().IsEnabledFor(logging.DEBUG) { - if err != nil { - yqlib.GetLogger().Debugf("error: %v", err) - } else { - yqlib.GetLogger().Debugf("error: %v, dir: %v", err, stat.IsDir()) - } - yqlib.GetLogger().Debugf("result: %v", result) - } - return result -} - -func processStdInArgs(args []string) []string { - stat, _ := os.Stdin.Stat() - pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0 - - // if we've been given a file, don't automatically - // read from stdin. - // this happens if there is more than one argument - // or only one argument and its a file - if nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) { - return args - } - - for _, arg := range args { - if arg == "-" { - return args - } - } - yqlib.GetLogger().Debugf("missing '-', adding it to the end") - - // we're piping from stdin, but there's no '-' arg - // lets add one to the end - return append(args, "-") -} - -func processArgs(originalArgs []string) (string, []string, error) { - expression := forceExpression - if expressionFile != "" { - expressionBytes, err := os.ReadFile(expressionFile) - if err != nil { - return "", nil, err - } - expression = string(expressionBytes) - } - - args := processStdInArgs(originalArgs) - yqlib.GetLogger().Debugf("processed args: %v", args) - if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) { - yqlib.GetLogger().Debug("assuming expression is '%v'", args[0]) - expression = args[0] - args = args[1:] - } - return expression, args, nil -} diff --git a/external/yq/cmd/version.go b/external/yq/cmd/version.go deleted file mode 100755 index f222e4c..0000000 --- a/external/yq/cmd/version.go +++ /dev/null @@ -1,49 +0,0 @@ -package cmd - -import ( - "fmt" - "strings" -) - -// The git commit that was compiled. This will be filled in by the compiler. -var ( - GitCommit string - GitDescribe string - - // Version is main version number that is being run at the moment. - Version = "4.30.1" - - // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) - // then it means that it is a final release. Otherwise, this is a pre-release - // such as "dev" (in development), "beta", "rc1", etc. - VersionPrerelease = "" -) - -// ProductName is the name of the product -const ProductName = "yq" - -// GetVersionDisplay composes the parts of the version in a way that's suitable -// for displaying to humans. -func GetVersionDisplay() string { - return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion()) -} - -func getHumanVersion() string { - version := Version - if GitDescribe != "" { - version = GitDescribe - } - - release := VersionPrerelease - if release != "" { - if !strings.Contains(version, release) { - version += fmt.Sprintf("-%s", release) - } - if GitCommit != "" { - version += fmt.Sprintf(" (%s)", GitCommit) - } - } - - // Strip off any single quotes added by the git information. - return strings.Replace(version, "'", "", -1) -} diff --git a/external/yq/cmd/version_test.go b/external/yq/cmd/version_test.go deleted file mode 100755 index 1ffc64a..0000000 --- a/external/yq/cmd/version_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package cmd - -import "testing" - -func TestGetVersionDisplay(t *testing.T) { - var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version - if VersionPrerelease != "" { - expectedVersion = expectedVersion + "-" + VersionPrerelease - } - expectedVersion = expectedVersion + "\n" - tests := []struct { - name string - want string - }{ - { - name: "Display Version", - want: expectedVersion, - }, - } - for _, tt := range tests { - if got := GetVersionDisplay(); got != tt.want { - t.Errorf("%q. GetVersionDisplay() = %v, want %v", tt.name, got, tt.want) - } - } -} - -func Test_getHumanVersion(t *testing.T) { - GitDescribe = "e42813d" - GitCommit = "e42813d+CHANGES" - var wanted string - if VersionPrerelease == "" { - wanted = GitDescribe - } else { - wanted = "e42813d-" + VersionPrerelease + " (e42813d+CHANGES)" - } - - tests := []struct { - name string - want string - }{ - { - name: "Git Variables defined", - want: wanted, - }, - } - for _, tt := range tests { - if got := getHumanVersion(); got != tt.want { - t.Errorf("%q. getHumanVersion() = %v, want %v", tt.name, got, tt.want) - } - } -} diff --git a/external/yq/debian/changelog b/external/yq/debian/changelog deleted file mode 100755 index 25c4b94..0000000 --- a/external/yq/debian/changelog +++ /dev/null @@ -1,210 +0,0 @@ -yq (4.16.2) focal; urgency=medium - - * Fixed with semicolon space issue - * Updating with documentation - * Added STDIN example to the top - * minor readme cleanup - * Help text tweak - * Fixed docker timeout - simplify docker builds - * New release with docker build fixes - * Updating to go 1.17 to fix CVE #944 - * Fix a typo in root.go - * Skip the tests if the nocheck Debian build option is specified - * Fixed select bug (#958) - * Sped up explode operator - * Slight performance improvement to context.ChildContext - * Speed up multiply - * Update README with recently added / changed options - * Make deepMatch report in linear time - * Removed leadingContentPreProcessing flag - header preprocessing is stable - * Revert "Removed leadingContentPreProcessing flag - header preprocessing is stable" - * Keep flag, it is needed in corner cases - * Updated Readme - * Man page - * Fixed expression parsing bug #970 - * Bumping go-lang, docker versions - * Added test release flow - * Updated github action release to generate man page - * Bumping version - * Removing no longer needed github action - * Added decoder op - * Fixed newline handling when decoding/encoding - * Fixed newline handling in encoder/decoder - * Can specify indent in encode ops - * Added group_by operator - * Added flatten operator - * Fixed flatten error message - * Improving docs - * Split printer - * Refactored command logic - * Fix JSON encoding removing null #985 - * Fixed acceptance tests - * gitbook - * Update document generation script - * Updating README - * Updating release instructions - * github action no longer uses data1.yml - * Create dependabot.yml - * Bump actions/create-release from 1.0.0 to 1.1.4 - * Bump actions/setup-go from 1 to 2.1.4 - * Bump github.com/goccy/go-yaml from 1.8.9 to 1.9.4 - * Bump github.com/jinzhu/copier from 0.2.8 to 0.3.2 - * Bump github.com/fatih/color from 1.10.0 to 1.13.0 - * Bump github.com/spf13/cobra from 1.1.3 to 1.2.1 - * Update dependabot.yml - * Update go.yml - * add build check to PRs - * Include secure as part of build process - * Fixing bad label in github action - * fixed printer test - * remove leading content indicator - * Fixed header preprocessing! - * lint : define golangci configuration file - * Update check.sh - * Load file acceptance test - * Minor improvement on handling front matter - * Improved load doc - * feature: detect MANPATh and install there - * Update install-man-page.sh - * simplify prod stage, move version label to action - * add labels, quote some values - * enable errorlint linter - * Added errorlint to devtools - * Added key operator - * Added more tests - * Fixing comments - * Attempt to fix golint problem - * Include version query for tools - * Clean up errored file? - * enable misspell linter - * updated readme - * update Golangci version to v1.43.0 - * gci linter - * Better merge array by key example - * Added credit for merge by array example - * Better formatting of merge arrays example - * Better merge example - * Add accessor for the yq logger instance (#1013) - * Fixed collect op when working with multiple nodes - * Added map, map_values - * Add support for Podman as well as Docker (#1026) - * Bump github.com/jinzhu/copier from 0.3.2 to 0.3.4 (#1027) - * Added csv, tsv output formats - * Added encoder tests - * Cleanup test - * Fixed docker permission issue #1014 - * Recording release notes for next release - * Assignment op no longer clobbers anchor (#1029) - * Added sort_by operator - * Improved error message - * Improved tips and tricks - * Report while filename failed to parse #1030 - * Added script for extracting checksums - * Improved extract-checksum.sh - * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039) - * enable more linters (#1043) - * Bump golang compiler #1037 - - -- Roberto Mier Escandon Tue, 21 Dec 2021 09:41:44 +0000 - -yq (4.13.0) focal; urgency=medium - - * New `with` operator for making multiple changes to a given path - * New `contains` operator, works like the `jq` equivalent - * Subtract operator now supports subtracting elements from arrays! - * Fixed Swapping values using variables #934 - * Github Action now properly supports multiline output #936, thanks @pjxiao - * Fixed missing closing bracket validation #932 - * Fix processing of hex numbers #929 - * Fixed alternative and union operator issues #930 - * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info. - * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919) - * pretty print yaml 1.1 compatibility (https://github.com/mikefarah/yq/issues/914) - - -- Roberto Mier Escandon Thu, 16 Sep 2021 20:58:30 +0200 - -yq (4.9.6) focal; urgency=medium - - * Added darwin/arm64 build, thanks @alecthomas - * Incremented docker alpine base version, thanks @da6d6i7-bronga - * Bug fix: multine expression - * Bug fix: special character - - -- Roberto Mier Escandon Tue, 29 Jun 2021 21:32:14 +0200 - -yq (3.3.2) focal; urgency=medium - - * Bug fix: existStatus bug (#459) - * Automatically makes a os temp directory if it does not exist (#461) - - -- Roberto Mier Escandon Fri, 07 Aug 2020 18:53:01 +0200 - -yq (3.3-0) focal; urgency=medium - - * You can control string styles (quotes) using the new --style flag - * String values now always have quotes when outputting to json - * Negative array indices now traverse the array backwards - * Added a --stripComments flag to print yaml without any comments - * Bumped go to version 1.14 - - -- Roberto Mier Escandon Thu, 30 Apr 2020 20:45:44 +0200 - -yq (3.1-2) eoan; urgency=medium - - * Bug fix: yq 3 was removing empty inline-style objects and arrays (#355) - * Bug fix: Merge option returned different output when switching order of - merging files(#347) - * Bug fix: Add new object to existing array object was failing in 3.1.1 (#361) - * Bug fix: yq 3 empty keys did not allow merging of values (#356) - * Bug fix: keys quoted during merge (#363) - * Bug fix: Correct length with wc -l (#362) - * Bug fix: Write to empty document removed path (#359) - - -- Roberto Mier Escandon Mon, 24 Feb 2020 20:31:58 +0100 - -yq (3.1-1) eoan; urgency=medium - - * Keeps yaml comments and formatting, can specify yaml tags when updating. - * Handles anchors - * Can print out matching paths and values when splatting - * JSON output works for all commands - * Yaml files with multiple documents are printed out as one JSON - document per line. - * Deep splat (**) to match arbitrary paths - * Update scripts file format has changed to be more powerful - * Reading and splatting, matching results are printed once per line - * Bugfixing - - -- Roberto Mier Escandon Tue, 11 Feb 2020 22:18:24 +0100 - -yq (2.2-1) bionic; urgency=medium - - * Added Windows support for the "--inplace" command flag - * Prefix now supports arrays - * Add prefix command - * Bump Alpine version to 3.8 - * Improved docker build process - * Lint fixes - * Build support for all linux architectures supported by gox - - -- Roberto Mier Escandon Sat, 19 Jan 2019 15:50:47 +0100 - -yq (2.1-0) bionic; urgency=medium - - * Ability to read multiple documents in a single file - * Ability to append list items instead of overwriting - - -- Roberto Mier Escandón Tue, 10 Jul 2018 14:02:42 +0200 - -yq (2.0-0) bionic; urgency=medium - - * Release 2.0.0 - - -- Roberto Mier Escandón Wed, 20 Jun 2018 10:29:53 +0200 - -yq (1.15-0) bionic; urgency=medium - - * Release 1.15 - - -- Roberto Mier Escandón Wed, 06 Jun 2018 11:32:03 +0200 - diff --git a/external/yq/debian/compat b/external/yq/debian/compat deleted file mode 100755 index f599e28..0000000 --- a/external/yq/debian/compat +++ /dev/null @@ -1 +0,0 @@ -10 diff --git a/external/yq/debian/control b/external/yq/debian/control deleted file mode 100755 index 625bfb3..0000000 --- a/external/yq/debian/control +++ /dev/null @@ -1,22 +0,0 @@ -Source: yq -Section: devel -Priority: optional -Maintainer: Roberto Mier Escandón -Build-Depends: debhelper (>=10), - golang-1.17-go, - pandoc, - rsync -Standards-Version: 4.1.4 -Homepage: https://github.com/mikefarah/yq.git -Vcs-Browser: https://github.com/mikefarah/yq.git -Vcs-Git: https://github.com/mikefarah/yq.git -XS-Go-Import-Path: github.com/mikefarah/yq -XSBC-Original-Maintainer: Roberto Mier Escandón - -Package: yq -Architecture: any -Depends: ${shlibs:Depends}, ${misc:Depends} -Description: lightweight and portable command-line YAML processor - . - The aim of the project is to be the - [jq](https://github.com/stedolan/jq) or sed of yaml files. diff --git a/external/yq/debian/copyright b/external/yq/debian/copyright deleted file mode 100755 index d3bbdc0..0000000 --- a/external/yq/debian/copyright +++ /dev/null @@ -1,24 +0,0 @@ -Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ -Upstream-Name: yq -Source: https://github.com/mikefarah/yq.git - -Files: * -Copyright: 2017 Mike Farah -License: Expat - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - . - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - . - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. \ No newline at end of file diff --git a/external/yq/debian/gbp.conf b/external/yq/debian/gbp.conf deleted file mode 100755 index cec628c..0000000 --- a/external/yq/debian/gbp.conf +++ /dev/null @@ -1,2 +0,0 @@ -[DEFAULT] -pristine-tar = True diff --git a/external/yq/debian/rules b/external/yq/debian/rules deleted file mode 100755 index 4c40d6d..0000000 --- a/external/yq/debian/rules +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/make -f -# -# Copyright (C) 2018-2021 Roberto Mier Escandón -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 3 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -PROJECT := yq -OWNER := mikefarah -REPO := github.com - -export DH_OPTIONS -export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT} -export GOROOT := /usr/local/go -export GOPATH := ${CURDIR}/_build -export GOBIN := ${GOPATH}/bin -export PATH := ${GOROOT}/bin:${GOBIN}:${PATH} -export GOCACHE := /tmp/gocache -export GOFLAGS := -mod=vendor -export GO111MODULE := on - -SRCDIR := ${GOPATH}/src/${DH_GOPKG} -DESTDIR := ${CURDIR}/debian/${PROJECT} -BINDIR := /usr/bin -MANDIR := /usr/share/man/man1/ -ASSETSDIR := /usr/share/${PROJECT} - -%: - dh $@ - -override_dh_auto_build: - mkdir -p ${SRCDIR} - mkdir -p ${GOBIN} - # copy project to local srcdir to build from there - rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR) - # build go code - ( \ - cd ${SRCDIR} && \ - go install -buildmode=pie ./... \ - ) - - # build man page - ( \ - cd ${SRCDIR} && \ - ./scripts/generate-man-page-md.sh && \ - ./scripts/generate-man-page.sh \ - ) - -override_dh_auto_test: -ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) - (cd ${SRCDIR} && go test -v ./...) -endif - -override_dh_auto_install: - cp ${GOBIN}/yq ${DESTDIR}/${BINDIR} - cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR} - chmod a+x ${DESTDIR}/${BINDIR}/yq - - # man - mkdir -p "${DESTDIR}"/"${MANDIR}" - cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \ - -override_dh_auto_clean: - dh_clean - rm -rf ${CURDIR}/_build diff --git a/external/yq/debian/source/format b/external/yq/debian/source/format deleted file mode 100755 index 89ae9db..0000000 --- a/external/yq/debian/source/format +++ /dev/null @@ -1 +0,0 @@ -3.0 (native) diff --git a/external/yq/debian/yq.dirs b/external/yq/debian/yq.dirs deleted file mode 100755 index ccf6e91..0000000 --- a/external/yq/debian/yq.dirs +++ /dev/null @@ -1,3 +0,0 @@ -usr/bin -usr/share/yq -usr/share/man/man1 \ No newline at end of file diff --git a/external/yq/examples/array.yaml b/external/yq/examples/array.yaml deleted file mode 100755 index 07b8808..0000000 --- a/external/yq/examples/array.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- [cat, dog, frog, cow] -- [apple, banana, grape, mango] \ No newline at end of file diff --git a/external/yq/examples/bad.yaml b/external/yq/examples/bad.yaml deleted file mode 100755 index 427823f..0000000 --- a/external/yq/examples/bad.yaml +++ /dev/null @@ -1,8 +0,0 @@ -b: - d: be gone - c: 2 - e: - - name: Billy Bob # comment over here - ---- -[123123 \ No newline at end of file diff --git a/external/yq/examples/base64.txt b/external/yq/examples/base64.txt deleted file mode 100755 index c74e092..0000000 --- a/external/yq/examples/base64.txt +++ /dev/null @@ -1 +0,0 @@ -bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u \ No newline at end of file diff --git a/external/yq/examples/data1-no-comments.yaml b/external/yq/examples/data1-no-comments.yaml deleted file mode 100755 index b0f017b..0000000 --- a/external/yq/examples/data1-no-comments.yaml +++ /dev/null @@ -1,4 +0,0 @@ -a: simple -b: [1, 2] -c: - test: 1 diff --git a/external/yq/examples/data1.yaml b/external/yq/examples/data1.yaml deleted file mode 100755 index 850ab07..0000000 --- a/external/yq/examples/data1.yaml +++ /dev/null @@ -1 +0,0 @@ -["foobar", "foobaz", "blarp"] \ No newline at end of file diff --git a/external/yq/examples/data2.yaml b/external/yq/examples/data2.yaml deleted file mode 100755 index fb5bcc6..0000000 --- a/external/yq/examples/data2.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# -------------------------------------------------- -# It's a test with comment -# -------------------------------------------------- -groups: - - name: d \ No newline at end of file diff --git a/external/yq/examples/data3.yaml b/external/yq/examples/data3.yaml deleted file mode 100755 index 22e8056..0000000 --- a/external/yq/examples/data3.yaml +++ /dev/null @@ -1,4 +0,0 @@ -a: "simple" # just the best -b: [1, 3] -c: - test: 1 \ No newline at end of file diff --git a/external/yq/examples/empty-no-comment.yaml b/external/yq/examples/empty-no-comment.yaml deleted file mode 100755 index e69de29..0000000 diff --git a/external/yq/examples/empty.yaml b/external/yq/examples/empty.yaml deleted file mode 100755 index 308ec1d..0000000 --- a/external/yq/examples/empty.yaml +++ /dev/null @@ -1 +0,0 @@ -# comment diff --git a/external/yq/examples/example.properties b/external/yq/examples/example.properties deleted file mode 100755 index 71af9cd..0000000 --- a/external/yq/examples/example.properties +++ /dev/null @@ -1,6 +0,0 @@ -# comments on values appear -person.name = Mike - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza \ No newline at end of file diff --git a/external/yq/examples/front-matter.yaml b/external/yq/examples/front-matter.yaml deleted file mode 100755 index 8ce5d4b..0000000 --- a/external/yq/examples/front-matter.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -a: apple -b: bannana ---- -hello there -apples: great \ No newline at end of file diff --git a/external/yq/examples/instruction_sample.yaml b/external/yq/examples/instruction_sample.yaml deleted file mode 100755 index 652e022..0000000 --- a/external/yq/examples/instruction_sample.yaml +++ /dev/null @@ -1,7 +0,0 @@ -- command: update - path: b.c - value: - #great - things: frog # wow! -- command: delete - path: b.d \ No newline at end of file diff --git a/external/yq/examples/leading-seperator.yaml b/external/yq/examples/leading-seperator.yaml deleted file mode 100755 index cf11014..0000000 --- a/external/yq/examples/leading-seperator.yaml +++ /dev/null @@ -1,2 +0,0 @@ ---- -a: test \ No newline at end of file diff --git a/external/yq/examples/merge-anchor.yaml b/external/yq/examples/merge-anchor.yaml deleted file mode 100755 index 61041bd..0000000 --- a/external/yq/examples/merge-anchor.yaml +++ /dev/null @@ -1,19 +0,0 @@ -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c - -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c - -foobarList: - b: foobarList_b - <<: [*foo,*bar] - c: foobarList_c - -foobar: - c: foobar_c - <<: *foo - thing: foobar_thing \ No newline at end of file diff --git a/external/yq/examples/mike.xml b/external/yq/examples/mike.xml deleted file mode 100755 index 4543a10..0000000 --- a/external/yq/examples/mike.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - -things - \ No newline at end of file diff --git a/external/yq/examples/mike2.xml b/external/yq/examples/mike2.xml deleted file mode 100755 index 0ad8095..0000000 --- a/external/yq/examples/mike2.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - cool - - ba2234r - bar2234233 - \ No newline at end of file diff --git a/external/yq/examples/multiline-text.yaml b/external/yq/examples/multiline-text.yaml deleted file mode 100755 index 8ecccac..0000000 --- a/external/yq/examples/multiline-text.yaml +++ /dev/null @@ -1,5 +0,0 @@ -test: | - abcdefg - hijklmno - - diff --git a/external/yq/examples/multiple_docs.yaml b/external/yq/examples/multiple_docs.yaml deleted file mode 100755 index cc94aad..0000000 --- a/external/yq/examples/multiple_docs.yaml +++ /dev/null @@ -1,18 +0,0 @@ -commonKey: first document -a: Easy! as one two three -b: - c: 2 - d: [3, 4] - e: - - name: fred - value: 3 - - name: sam - value: 4 ---- -commonKey: second document -another: - document: here ---- -commonKey: third document -wow: - - here is another diff --git a/external/yq/examples/multiple_docs_small.yaml b/external/yq/examples/multiple_docs_small.yaml deleted file mode 100755 index 86035cc..0000000 --- a/external/yq/examples/multiple_docs_small.yaml +++ /dev/null @@ -1,7 +0,0 @@ -a: Easy! as one two three ---- -another: - document: here ---- -- 1 -- 2 \ No newline at end of file diff --git a/external/yq/examples/numbered_keys.yml b/external/yq/examples/numbered_keys.yml deleted file mode 100755 index 9a15c7e..0000000 --- a/external/yq/examples/numbered_keys.yml +++ /dev/null @@ -1,2 +0,0 @@ -5: - 6: camel! \ No newline at end of file diff --git a/external/yq/examples/order.yaml b/external/yq/examples/order.yaml deleted file mode 100755 index b3d896b..0000000 --- a/external/yq/examples/order.yaml +++ /dev/null @@ -1,2 +0,0 @@ -version: 3 -application: MyApp \ No newline at end of file diff --git a/external/yq/examples/order.yml b/external/yq/examples/order.yml deleted file mode 100755 index 2dc6204..0000000 --- a/external/yq/examples/order.yml +++ /dev/null @@ -1,6 +0,0 @@ -version: '2' -services: - test: - image: ubuntu:14.04 - stdin_open: true - tty: true \ No newline at end of file diff --git a/external/yq/examples/sample.json b/external/yq/examples/sample.json deleted file mode 100755 index 5a35f97..0000000 --- a/external/yq/examples/sample.json +++ /dev/null @@ -1 +0,0 @@ -{"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]},"ab":"must appear last"} diff --git a/external/yq/examples/sample.yaml b/external/yq/examples/sample.yaml deleted file mode 100755 index 9b925d7..0000000 --- a/external/yq/examples/sample.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Some doc - -a: true -b: - c: 2 - d: [3, 4, 5] - e: - - name: fred - value: 3 - - name: sam - value: 4 \ No newline at end of file diff --git a/external/yq/examples/sample_array.yaml b/external/yq/examples/sample_array.yaml deleted file mode 100755 index 3a26a2e..0000000 --- a/external/yq/examples/sample_array.yaml +++ /dev/null @@ -1 +0,0 @@ -[1,2,3] \ No newline at end of file diff --git a/external/yq/examples/sample_array_2.yaml b/external/yq/examples/sample_array_2.yaml deleted file mode 100755 index e83ee7a..0000000 --- a/external/yq/examples/sample_array_2.yaml +++ /dev/null @@ -1,2 +0,0 @@ -- 4 -- 5 \ No newline at end of file diff --git a/external/yq/examples/sample_objects.csv b/external/yq/examples/sample_objects.csv deleted file mode 100755 index 84ae60b..0000000 --- a/external/yq/examples/sample_objects.csv +++ /dev/null @@ -1,3 +0,0 @@ -name,numberOfCats,likesApples,height -,1,true,168.8 -Samantha's Rabbit,2,false,-188.8 \ No newline at end of file diff --git a/external/yq/examples/sample_text.yaml b/external/yq/examples/sample_text.yaml deleted file mode 100755 index 45b983b..0000000 --- a/external/yq/examples/sample_text.yaml +++ /dev/null @@ -1 +0,0 @@ -hi diff --git a/external/yq/examples/simple-anchor-exploded.yaml b/external/yq/examples/simple-anchor-exploded.yaml deleted file mode 100755 index 7cf0d51..0000000 --- a/external/yq/examples/simple-anchor-exploded.yaml +++ /dev/null @@ -1,5 +0,0 @@ -foo: - a: 1 - -foobar: - a: 1 \ No newline at end of file diff --git a/external/yq/examples/simple-anchor.yaml b/external/yq/examples/simple-anchor.yaml deleted file mode 100755 index 35218bb..0000000 --- a/external/yq/examples/simple-anchor.yaml +++ /dev/null @@ -1,5 +0,0 @@ -foo: &foo - a: 1 - -foobar: - <<: *foo \ No newline at end of file diff --git a/external/yq/examples/small.properties b/external/yq/examples/small.properties deleted file mode 100755 index 7a75a2e..0000000 --- a/external/yq/examples/small.properties +++ /dev/null @@ -1 +0,0 @@ -this.is = a properties file \ No newline at end of file diff --git a/external/yq/examples/small.xml b/external/yq/examples/small.xml deleted file mode 100755 index 8bec459..0000000 --- a/external/yq/examples/small.xml +++ /dev/null @@ -1 +0,0 @@ -is some xml \ No newline at end of file diff --git a/external/yq/examples/thing.yml b/external/yq/examples/thing.yml deleted file mode 100755 index 582ebae..0000000 --- a/external/yq/examples/thing.yml +++ /dev/null @@ -1,2 +0,0 @@ -a: apple is included -b: cool. \ No newline at end of file diff --git a/external/yq/github-action/Dockerfile b/external/yq/github-action/Dockerfile deleted file mode 100755 index dfc9086..0000000 --- a/external/yq/github-action/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM mikefarah/yq:4 - -COPY entrypoint.sh /entrypoint.sh - -# github action recommendation is to run as root. -# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user -USER root - -ENTRYPOINT ["/entrypoint.sh"] diff --git a/external/yq/github-action/entrypoint.sh b/external/yq/github-action/entrypoint.sh deleted file mode 100755 index a328fbd..0000000 --- a/external/yq/github-action/entrypoint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh -l -set -e -echo "::debug::\$cmd: $1" -RESULT=$(eval "$1") -echo "::debug::\$RESULT: $RESULT" -# updating from -# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281 -# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter -delimiter=$(cat /proc/sys/kernel/random/uuid) -echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}" -echo "${RESULT}" >> "${GITHUB_OUTPUT}" -echo "${delimiter}" >> "${GITHUB_OUTPUT}" diff --git a/external/yq/go.sum b/external/yq/go.sum deleted file mode 100755 index e4ca963..0000000 --- a/external/yq/go.sum +++ /dev/null @@ -1,83 +0,0 @@ -github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg= -github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= -github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg= -github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo= -github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM= -github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= -github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= -github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg= -github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-yaml v1.9.6 h1:KhAu1zf9JXnm3vbG49aDE0E5uEBUsM4uwD31/58ZWyI= -github.com/goccy/go-yaml v1.9.6/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE= -github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0= -golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew= -golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= -gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/external/yq/how-it-works.md b/external/yq/how-it-works.md deleted file mode 100755 index 51a41fc..0000000 --- a/external/yq/how-it-works.md +++ /dev/null @@ -1,129 +0,0 @@ -# How it works - -In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval). - -Lets look at a couple of examples. - -## Simple assignment example - -Given a document like: - -```yaml -a: cat -b: dog -``` - -with an expression: - -``` -.a = .b -``` - -Like math expressions - operator precedence is important. - -The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. - -It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node - -```yaml -cat -``` - -Sidenote: this node holds not only its value 'cat', but comments and metadata too, including path and parent information. - -The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node - -```yaml -dog -``` - -Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context: - -```yaml -a: dog -b: dog -``` - - -## Complex assignment, operator precedence rules - -Just like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. - -Most of the time, this is intuitively what you'd want, for instance `.a = "cat" | .b = "dog"` is effectively: `(.a = "cat") | (.b = "dog")`. - -However, this is not always the case, particularly if you have a complex LHS or RHS expression, for instance if you want to select particular nodes to update. - -Lets say you had: - -```yaml -- name: bob - fruit: apple -- name: sally - fruit: orange - -``` - -Lets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is: -`.[] | select(.name == "sally") | .fruit = "mango"`. - -Because `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned: - -```yaml -name: sally -fruit: mango -``` - -To properly update this yaml, you will need to use brackets (think BODMAS from maths) and wrap the entire LHS: -`(.[] | select(.name == "sally") | .fruit) = "mango"` - - -Now that entire LHS expression is passed to the 'assign' (`=`) operator, and the yaml is correctly updated and returned: - - -```yaml -- name: bob - fruit: apple -- name: sally - fruit: mango - -``` - -## Relative update (e.g. `|=`) -There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression. - -In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example. - -Given a document like: - -```yaml -a: 1 -b: thing -``` - -with an expression: - -``` -.a |= . + 1 -``` - -Similar to the `=` operator, `|=` takes two operands, the LHS and RHS. - -It pipes the current context (the whole document) through the LHS expression of `.a` to get the node value: - -``` -1 -``` - -Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield: - - -``` -2 -``` - -The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context: - -```yaml -a: 2 -b: thing -``` diff --git a/external/yq/mkdocs.yml b/external/yq/mkdocs.yml deleted file mode 100755 index 9896ee0..0000000 --- a/external/yq/mkdocs.yml +++ /dev/null @@ -1,7 +0,0 @@ -docs_dir: mkdocs -site_dir: docs -site_name: Yq -theme: 'material' -repo_name: 'mikefarah/yq' -repo_url: 'https://github.com/mikefarah/yq' - diff --git a/external/yq/pkg/yqlib/all_at_once_evaluator.go b/external/yq/pkg/yqlib/all_at_once_evaluator.go deleted file mode 100755 index 4dbed3d..0000000 --- a/external/yq/pkg/yqlib/all_at_once_evaluator.go +++ /dev/null @@ -1,83 +0,0 @@ -package yqlib - -import ( - "container/list" - - yaml "gopkg.in/yaml.v3" -) - -// A yaml expression evaluator that runs the expression once against all files/nodes in memory. -type Evaluator interface { - EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error - - // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes - EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) - - // EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes - EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error) -} - -type allAtOnceEvaluator struct { - treeNavigator DataTreeNavigator -} - -func NewAllAtOnceEvaluator() Evaluator { - InitExpressionParser() - return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()} -} - -func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) { - inputCandidates := list.New() - for _, node := range nodes { - inputCandidates.PushBack(&CandidateNode{Node: node}) - } - return e.EvaluateCandidateNodes(expression, inputCandidates) -} - -func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) { - node, err := ExpressionParser.ParseExpression(expression) - if err != nil { - return nil, err - } - context, err := e.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputCandidates}, node) - if err != nil { - return nil, err - } - return context.MatchingNodes, nil -} - -func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { - fileIndex := 0 - - var allDocuments = list.New() - for _, filename := range filenames { - reader, err := readStream(filename) - if err != nil { - return err - } - - fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) - if err != nil { - return err - } - allDocuments.PushBackList(fileDocuments) - fileIndex = fileIndex + 1 - } - - if allDocuments.Len() == 0 { - candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - LeadingContent: "", - } - allDocuments.PushBack(candidateNode) - } - - matches, err := e.EvaluateCandidateNodes(expression, allDocuments) - if err != nil { - return err - } - return printer.PrintResults(matches) -} diff --git a/external/yq/pkg/yqlib/all_at_once_evaluator_test.go b/external/yq/pkg/yqlib/all_at_once_evaluator_test.go deleted file mode 100755 index 2e9abc4..0000000 --- a/external/yq/pkg/yqlib/all_at_once_evaluator_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package yqlib - -import ( - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -var evaluateNodesScenario = []expressionScenario{ - { - document: `a: hello`, - expression: `.a`, - expected: []string{ - "D0, P[a], (!!str)::hello\n", - }, - }, - { - document: `a: hello`, - expression: `.`, - expected: []string{ - "D0, P[], (doc)::a: hello\n", - }, - }, - { - document: `- a: "yes"`, - expression: `.[] | has("a")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - }, - }, -} - -func TestAllAtOnceEvaluateNodes(t *testing.T) { - var evaluator = NewAllAtOnceEvaluator() - for _, tt := range evaluateNodesScenario { - node := test.ParseData(tt.document) - list, _ := evaluator.EvaluateNodes(tt.expression, &node) - test.AssertResultComplex(t, tt.expected, resultsToString(t, list)) - } -} diff --git a/external/yq/pkg/yqlib/candidate_node.go b/external/yq/pkg/yqlib/candidate_node.go deleted file mode 100755 index 61675c8..0000000 --- a/external/yq/pkg/yqlib/candidate_node.go +++ /dev/null @@ -1,189 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "strings" - - "github.com/jinzhu/copier" - yaml "gopkg.in/yaml.v3" -) - -type CandidateNode struct { - Node *yaml.Node // the actual node - Parent *CandidateNode // parent node - Key *yaml.Node // node key, if this is a value from a map (or index in an array) - - LeadingContent string - TrailingContent string - - Path []interface{} /// the path we took to get to this node - Document uint // the document index of this node - Filename string - FileIndex int - // when performing op against all nodes given, this will treat all the nodes as one - // (e.g. top level cross document merge). This property does not propagate to child nodes. - EvaluateTogether bool - IsMapKey bool -} - -func (n *CandidateNode) GetKey() string { - keyPrefix := "" - if n.IsMapKey { - keyPrefix = "key-" - } - return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path) -} - -func (n *CandidateNode) GetNiceTag() string { - return unwrapDoc(n.Node).Tag -} - -func (n *CandidateNode) GetNicePath() string { - if n.Path != nil && len(n.Path) >= 0 { - pathStr := make([]string, len(n.Path)) - for i, v := range n.Path { - pathStr[i] = fmt.Sprintf("%v", v) - } - return strings.Join(pathStr, ".") - } - return "" -} - -func (n *CandidateNode) AsList() *list.List { - elMap := list.New() - elMap.PushBack(n) - return elMap -} - -func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode { - var value interface{} - if key != nil { - value = key.Value - } - return &CandidateNode{ - Node: node, - Path: n.createChildPath(value), - Parent: n, - Key: key, - Document: n.Document, - Filename: n.Filename, - FileIndex: n.FileIndex, - } -} - -func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode { - return &CandidateNode{ - Node: node, - Path: n.createChildPath(index), - Parent: n, - Key: &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"}, - Document: n.Document, - Filename: n.Filename, - FileIndex: n.FileIndex, - } -} - -func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode { - return &CandidateNode{ - Node: node, - Path: n.createChildPath(nil), - Parent: n.Parent, - Key: n.Key, - IsMapKey: n.IsMapKey, - Document: n.Document, - Filename: n.Filename, - FileIndex: n.FileIndex, - } -} - -func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode { - replacement := n.CreateReplacement(node) - replacement.LeadingContent = n.LeadingContent - replacement.TrailingContent = n.TrailingContent - return replacement -} - -func (n *CandidateNode) createChildPath(path interface{}) []interface{} { - if path == nil { - newPath := make([]interface{}, len(n.Path)) - copy(newPath, n.Path) - return newPath - } - - //don't use append as they may actually modify the path of the orignal node! - newPath := make([]interface{}, len(n.Path)+1) - copy(newPath, n.Path) - newPath[len(n.Path)] = path - return newPath -} - -func (n *CandidateNode) Copy() (*CandidateNode, error) { - clone := &CandidateNode{} - err := copier.Copy(clone, n) - if err != nil { - return nil, err - } - clone.Node = deepClone(n.Node) - return clone, nil -} - -// updates this candidate from the given candidate node -func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) { - - // if this is an empty map or empty array, use the style of other node. - if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) || - // if the tag has changed (e.g. from str to bool) - (guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) { - n.Node.Style = other.Node.Style - } - - n.Node.Content = deepCloneContent(other.Node.Content) - n.Node.Kind = other.Node.Kind - n.Node.Value = other.Node.Value - - n.UpdateAttributesFrom(other, prefs) - -} - -func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) { - log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey()) - if n.Node.Kind != other.Node.Kind { - // clear out the contents when switching to a different type - // e.g. map to array - n.Node.Content = make([]*yaml.Node, 0) - n.Node.Value = "" - } - n.Node.Kind = other.Node.Kind - - // don't clobber custom tags... - if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" { - n.Node.Tag = other.Node.Tag - } - - n.Node.Alias = other.Node.Alias - - if !prefs.DontOverWriteAnchor { - n.Node.Anchor = other.Node.Anchor - } - - // merge will pickup the style of the new thing - // when autocreating nodes - - if n.Node.Style == 0 { - n.Node.Style = other.Node.Style - } - - if other.Node.FootComment != "" { - n.Node.FootComment = other.Node.FootComment - } - if other.TrailingContent != "" { - n.TrailingContent = other.TrailingContent - } - if other.Node.HeadComment != "" { - n.Node.HeadComment = other.Node.HeadComment - } - if other.Node.LineComment != "" { - n.Node.LineComment = other.Node.LineComment - } -} diff --git a/external/yq/pkg/yqlib/color_print.go b/external/yq/pkg/yqlib/color_print.go deleted file mode 100755 index 1128e85..0000000 --- a/external/yq/pkg/yqlib/color_print.go +++ /dev/null @@ -1,61 +0,0 @@ -package yqlib - -import ( - "fmt" - "io" - - "github.com/fatih/color" - "github.com/goccy/go-yaml/lexer" - "github.com/goccy/go-yaml/printer" -) - -// Thanks @risentveber! - -const escape = "\x1b" - -func format(attr color.Attribute) string { - return fmt.Sprintf("%s[%dm", escape, attr) -} - -func colorizeAndPrint(yamlBytes []byte, writer io.Writer) error { - tokens := lexer.Tokenize(string(yamlBytes)) - var p printer.Printer - p.Bool = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgHiMagenta), - Suffix: format(color.Reset), - } - } - p.Number = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgHiMagenta), - Suffix: format(color.Reset), - } - } - p.MapKey = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgCyan), - Suffix: format(color.Reset), - } - } - p.Anchor = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgHiYellow), - Suffix: format(color.Reset), - } - } - p.Alias = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgHiYellow), - Suffix: format(color.Reset), - } - } - p.String = func() *printer.Property { - return &printer.Property{ - Prefix: format(color.FgGreen), - Suffix: format(color.Reset), - } - } - _, err := writer.Write([]byte(p.PrintTokens(tokens) + "\n")) - return err -} diff --git a/external/yq/pkg/yqlib/compare_operators.go b/external/yq/pkg/yqlib/compare_operators.go deleted file mode 100755 index 63323e3..0000000 --- a/external/yq/pkg/yqlib/compare_operators.go +++ /dev/null @@ -1,133 +0,0 @@ -package yqlib - -import ( - "fmt" - "strconv" - "time" - - yaml "gopkg.in/yaml.v3" -) - -type compareTypePref struct { - OrEqual bool - Greater bool -} - -func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- compareOperator") - prefs := expressionNode.Operation.Preferences.(compareTypePref) - return crossFunction(d, context.ReadOnlyClone(), expressionNode, compare(prefs), true) -} - -func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - log.Debugf("-- compare cross function") - if lhs == nil && rhs == nil { - owner := &CandidateNode{} - return createBooleanCandidate(owner, prefs.OrEqual), nil - } else if lhs == nil { - log.Debugf("lhs nil, but rhs is not") - return createBooleanCandidate(rhs, false), nil - } else if rhs == nil { - log.Debugf("rhs nil, but rhs is not") - return createBooleanCandidate(lhs, false), nil - } - - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - - switch lhs.Node.Kind { - case yaml.MappingNode: - return nil, fmt.Errorf("maps not yet supported for comparison") - case yaml.SequenceNode: - return nil, fmt.Errorf("arrays not yet supported for comparison") - default: - if rhs.Node.Kind != yaml.ScalarNode { - return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhs.Node.Tag) - } - target := lhs.CreateReplacement(&yaml.Node{}) - boolV, err := compareScalars(context, prefs, lhs.Node, rhs.Node) - - return createBooleanCandidate(target, boolV), err - } - } -} - -func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - lhsTime, err := parseDateTime(layout, lhs.Value) - if err != nil { - return false, err - } - - rhsTime, err := parseDateTime(layout, rhs.Value) - if err != nil { - return false, err - } - - if prefs.OrEqual && lhsTime.Equal(rhsTime) { - return true, nil - } - if prefs.Greater { - return lhsTime.After(rhsTime), nil - } - return lhsTime.Before(rhsTime), nil - -} - -func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - lhsTag := guessTagFromCustomType(lhs) - rhsTag := guessTagFromCustomType(rhs) - - isDateTime := lhs.Tag == "!!timestamp" - // if the lhs is a string, it might be a timestamp in a custom format. - if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { - _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) - isDateTime = err == nil - } - if isDateTime { - return compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs) - } else if lhsTag == "!!int" && rhsTag == "!!int" { - _, lhsNum, err := parseInt64(lhs.Value) - if err != nil { - return false, err - } - _, rhsNum, err := parseInt64(rhs.Value) - if err != nil { - return false, err - } - - if prefs.OrEqual && lhsNum == rhsNum { - return true, nil - } - if prefs.Greater { - return lhsNum > rhsNum, nil - } - return lhsNum < rhsNum, nil - } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { - lhsNum, err := strconv.ParseFloat(lhs.Value, 64) - if err != nil { - return false, err - } - rhsNum, err := strconv.ParseFloat(rhs.Value, 64) - if err != nil { - return false, err - } - if prefs.OrEqual && lhsNum == rhsNum { - return true, nil - } - if prefs.Greater { - return lhsNum > rhsNum, nil - } - return lhsNum < rhsNum, nil - } else if lhsTag == "!!str" && rhsTag == "!!str" { - if prefs.OrEqual && lhs.Value == rhs.Value { - return true, nil - } - if prefs.Greater { - return lhs.Value > rhs.Value, nil - } - return lhs.Value < rhs.Value, nil - } - - return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag) -} diff --git a/external/yq/pkg/yqlib/compare_operators_test.go b/external/yq/pkg/yqlib/compare_operators_test.go deleted file mode 100755 index bf07247..0000000 --- a/external/yq/pkg/yqlib/compare_operators_test.go +++ /dev/null @@ -1,371 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var compareOperatorScenarios = []expressionScenario{ - // ints, not equal - { - description: "Compare numbers (>)", - document: "a: 5\nb: 4", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5\nb: 4", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - description: "Compare integers (>=)", - document: "a: 5\nb: 4", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5\nb: 4", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - - // ints, equal - { - skipDoc: true, - description: "Compare equal numbers (>)", - document: "a: 5\nb: 5", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 5\nb: 5", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - description: "Compare equal numbers (>=)", - document: "a: 5\nb: 5", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5\nb: 5", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - - // floats, not equal - { - skipDoc: true, - document: "a: 5.2\nb: 4.1", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5.2\nb: 4.1", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 5.2\nb: 4.1", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5.5\nb: 4.1", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - - // floats, equal - { - skipDoc: true, - document: "a: 5.5\nb: 5.5", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 5.5\nb: 5.5", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 5.1\nb: 5.1", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 5.1\nb: 5.1", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - - // strings, not equal - { - description: "Compare strings", - subdescription: "Compares strings by their bytecode.", - document: "a: zoo\nb: apple", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: zoo\nb: apple", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: zoo\nb: apple", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: zoo\nb: apple", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - - // strings, equal - { - skipDoc: true, - document: "a: cat\nb: cat", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: cat", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: cat", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: cat", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - - // datetime, not equal - { - description: "Compare date times", - subdescription: "You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - - // datetime, equal - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::true\n", - }, - }, - // both null - { - description: "Both sides are null: > is false", - expression: ".a > .b", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - expression: ".a < .b", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Both sides are null: >= is true", - expression: ".a >= .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - expression: ".a <= .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - - // one null - { - skipDoc: true, - description: "One side is null: > is false", - document: `a: 5`, - expression: ".a > .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".a < .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - description: "One side is null: >= is false", - document: `a: 5`, - expression: ".a >= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".a <= .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".b <= .a", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: 5`, - expression: ".b < .a", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, -} - -func TestCompareOperatorScenarios(t *testing.T) { - for _, tt := range compareOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "compare", compareOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/context.go b/external/yq/pkg/yqlib/context.go deleted file mode 100755 index 17cddb3..0000000 --- a/external/yq/pkg/yqlib/context.go +++ /dev/null @@ -1,122 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "time" - - "github.com/jinzhu/copier" - logging "gopkg.in/op/go-logging.v1" -) - -type Context struct { - MatchingNodes *list.List - Variables map[string]*list.List - DontAutoCreate bool - datetimeLayout string -} - -func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context { - list := list.New() - list.PushBack(candidate) - newContext := n.ChildContext(list) - newContext.DontAutoCreate = true - return newContext -} - -func (n *Context) SingleChildContext(candidate *CandidateNode) Context { - list := list.New() - list.PushBack(candidate) - return n.ChildContext(list) -} - -func (n *Context) SetDateTimeLayout(newDateTimeLayout string) { - n.datetimeLayout = newDateTimeLayout -} - -func (n *Context) GetDateTimeLayout() string { - if n.datetimeLayout != "" { - return n.datetimeLayout - } - return time.RFC3339 -} - -func (n *Context) GetVariable(name string) *list.List { - if n.Variables == nil { - return nil - } - return n.Variables[name] -} - -func (n *Context) SetVariable(name string, value *list.List) { - if n.Variables == nil { - n.Variables = make(map[string]*list.List) - } - n.Variables[name] = value -} - -func (n *Context) ChildContext(results *list.List) Context { - clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout} - clone.Variables = make(map[string]*list.List) - if len(n.Variables) > 0 { - err := copier.Copy(&clone.Variables, n.Variables) - if err != nil { - log.Error("Error cloning context :(") - panic(err) - } - } - clone.MatchingNodes = results - return clone -} - -func (n *Context) ToString() string { - if !log.IsEnabledFor(logging.DEBUG) { - return "" - } - result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate) - return result + NodesToString(n.MatchingNodes) -} - -func (n *Context) DeepClone() Context { - clone := Context{} - err := copier.Copy(&clone, n) - // copier doesn't do lists properly for some reason - clone.MatchingNodes = list.New() - for el := n.MatchingNodes.Front(); el != nil; el = el.Next() { - clonedNode, err := el.Value.(*CandidateNode).Copy() - if err != nil { - log.Error("Error cloning context :(") - panic(err) - } - clone.MatchingNodes.PushBack(clonedNode) - } - - if err != nil { - log.Error("Error cloning context :(") - panic(err) - } - return clone -} - -func (n *Context) Clone() Context { - clone := Context{} - err := copier.Copy(&clone, n) - - if err != nil { - log.Error("Error cloning context :(") - panic(err) - } - return clone -} - -func (n *Context) ReadOnlyClone() Context { - clone := n.Clone() - clone.DontAutoCreate = true - return clone -} - -func (n *Context) WritableClone() Context { - clone := n.Clone() - clone.DontAutoCreate = false - return clone -} diff --git a/external/yq/pkg/yqlib/csv_test.go b/external/yq/pkg/yqlib/csv_test.go deleted file mode 100755 index 45160ca..0000000 --- a/external/yq/pkg/yqlib/csv_test.go +++ /dev/null @@ -1,282 +0,0 @@ -package yqlib - -import ( - "bufio" - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -const csvSimple = `name,numberOfCats,likesApples,height -Gary,1,true,168.8 -Samantha's Rabbit,2,false,-188.8 -` -const csvMissing = `name,numberOfCats,likesApples,height -,null,,168.8 -` -const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height -Gary,3,true,168.8 -Samantha's Rabbit,2,false,-188.8 -` - -const csvSimpleShort = `Name,Number of Cats -Gary,1 -Samantha's Rabbit,2 -` - -const tsvSimple = `name numberOfCats likesApples height -Gary 1 true 168.8 -Samantha's Rabbit 2 false -188.8 -` - -const expectedYamlFromCSV = `- name: Gary - numberOfCats: 1 - likesApples: true - height: 168.8 -- name: Samantha's Rabbit - numberOfCats: 2 - likesApples: false - height: -188.8 -` - -const expectedYamlFromCSVMissingData = `- name: Gary - numberOfCats: 1 - height: 168.8 -- name: Samantha's Rabbit - height: -188.8 - likesApples: false -` - -const csvSimpleMissingData = `name,numberOfCats,height -Gary,1,168.8 -Samantha's Rabbit,,-188.8 -` - -const csvTestSimpleYaml = `- [i, like, csv] -- [because, excel, is, cool]` - -const expectedSimpleCsv = `i,like,csv -because,excel,is,cool -` - -const tsvTestExpectedSimpleCsv = `i like csv -because excel is cool -` - -var csvScenarios = []formatScenario{ - { - description: "Encode CSV simple", - input: csvTestSimpleYaml, - expected: expectedSimpleCsv, - scenarioType: "encode-csv", - }, - { - description: "Encode TSV simple", - input: csvTestSimpleYaml, - expected: tsvTestExpectedSimpleCsv, - scenarioType: "encode-tsv", - }, - { - description: "Encode Empty", - skipDoc: true, - input: `[]`, - expected: "", - scenarioType: "encode-csv", - }, - { - description: "Comma in value", - skipDoc: true, - input: `["comma, in, value", things]`, - expected: "\"comma, in, value\",things\n", - scenarioType: "encode-csv", - }, - { - description: "Encode array of objects to csv", - input: expectedYamlFromCSV, - expected: csvSimple, - scenarioType: "encode-csv", - }, - { - description: "Encode array of objects to custom csv format", - subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.", - input: expectedYamlFromCSV, - expected: csvSimpleShort, - expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`, - scenarioType: "encode-csv", - }, - { - description: "Encode array of objects to csv - missing fields behaviour", - subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank", - input: expectedYamlFromCSVMissingData, - expected: csvSimpleMissingData, - scenarioType: "encode-csv", - }, - { - description: "decode csv missing", - skipDoc: true, - input: csvMissing, - expected: csvMissing, - scenarioType: "roundtrip-csv", - }, - { - description: "Parse CSV into an array of objects", - subdescription: "First row is assumed to be the header row.", - input: csvSimple, - expected: expectedYamlFromCSV, - scenarioType: "decode-csv-object", - }, - { - description: "Parse TSV into an array of objects", - subdescription: "First row is assumed to be the header row.", - input: tsvSimple, - expected: expectedYamlFromCSV, - scenarioType: "decode-tsv-object", - }, - { - description: "Round trip", - input: csvSimple, - expected: expectedUpdatedSimpleCsv, - expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`, - scenarioType: "roundtrip-csv", - }, -} - -func testCSVScenario(t *testing.T, s formatScenario) { - switch s.scenarioType { - case "encode-csv": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description) - case "encode-tsv": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description) - case "decode-csv-object": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "decode-tsv-object": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "roundtrip-csv": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description) - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType)) - writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v sample.%v\n```\n", formatType, formatType)) - writeOrPanic(w, "will output\n") - - separator := ',' - if formatType == "tsv" { - separator = '\t' - } - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", - mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))), - ) -} - -func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v '%v' sample.yml\n```\n", formatType, expression)) - } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v sample.yml\n```\n", formatType)) - } - writeOrPanic(w, "will output\n") - - separator := ',' - if formatType == "tsv" { - separator = '\t' - } - - writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, - mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))), - ) -} - -func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType)) - writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v '%v' sample.%v\n```\n", formatType, formatType, expression, formatType)) - } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v sample.%v\n```\n", formatType, formatType, formatType)) - } - writeOrPanic(w, "will output\n") - - separator := ',' - if formatType == "tsv" { - separator = '\t' - } - - writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, - mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))), - ) -} - -func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) { - s := i.(formatScenario) - if s.skipDoc { - return - } - switch s.scenarioType { - case "encode-csv": - documentCSVEncodeScenario(w, s, "csv") - case "encode-tsv": - documentCSVEncodeScenario(w, s, "tsv") - case "decode-csv-object": - documentCSVDecodeObjectScenario(w, s, "csv") - case "decode-tsv-object": - documentCSVDecodeObjectScenario(w, s, "tsv") - case "roundtrip-csv": - documentCSVRoundTripScenario(w, s, "csv") - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func TestCSVScenarios(t *testing.T) { - for _, tt := range csvScenarios { - testCSVScenario(t, tt) - } - genericScenarios := make([]interface{}, len(csvScenarios)) - for i, s := range csvScenarios { - genericScenarios[i] = s - } - documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario) -} diff --git a/external/yq/pkg/yqlib/data_tree_navigator.go b/external/yq/pkg/yqlib/data_tree_navigator.go deleted file mode 100755 index 13659ea..0000000 --- a/external/yq/pkg/yqlib/data_tree_navigator.go +++ /dev/null @@ -1,41 +0,0 @@ -package yqlib - -import ( - "fmt" - - logging "gopkg.in/op/go-logging.v1" -) - -type DataTreeNavigator interface { - // given the context and a expressionNode, - // this will process the against the given expressionNode and return - // a new context of matching candidates - GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) -} - -type dataTreeNavigator struct { -} - -func NewDataTreeNavigator() DataTreeNavigator { - return &dataTreeNavigator{} -} - -func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) { - if expressionNode == nil { - log.Debugf("getMatchingNodes - nothing to do") - return context, nil - } - log.Debugf("Processing Op: %v", expressionNode.Operation.toString()) - if log.IsEnabledFor(logging.DEBUG) { - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - log.Debug(NodeToString(el.Value.(*CandidateNode))) - } - } - log.Debug(">>") - handler := expressionNode.Operation.OperationType.Handler - if handler != nil { - return handler(d, context, expressionNode) - } - return Context{}, fmt.Errorf("Unknown operator %v", expressionNode.Operation.OperationType) - -} diff --git a/external/yq/pkg/yqlib/decoder.go b/external/yq/pkg/yqlib/decoder.go deleted file mode 100755 index 904b65c..0000000 --- a/external/yq/pkg/yqlib/decoder.go +++ /dev/null @@ -1,42 +0,0 @@ -package yqlib - -import ( - "fmt" - "io" -) - -type InputFormat uint - -const ( - YamlInputFormat = 1 << iota - XMLInputFormat - PropertiesInputFormat - Base64InputFormat - JsonInputFormat - CSVObjectInputFormat - TSVObjectInputFormat -) - -type Decoder interface { - Init(reader io.Reader) error - Decode() (*CandidateNode, error) -} - -func InputFormatFromString(format string) (InputFormat, error) { - switch format { - case "yaml", "y": - return YamlInputFormat, nil - case "xml", "x": - return XMLInputFormat, nil - case "props", "p": - return PropertiesInputFormat, nil - case "json", "ndjson", "j": - return JsonInputFormat, nil - case "csv", "c": - return CSVObjectInputFormat, nil - case "tsv", "t": - return TSVObjectInputFormat, nil - default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format) - } -} diff --git a/external/yq/pkg/yqlib/decoder_base64.go b/external/yq/pkg/yqlib/decoder_base64.go deleted file mode 100755 index c4868fd..0000000 --- a/external/yq/pkg/yqlib/decoder_base64.go +++ /dev/null @@ -1,57 +0,0 @@ -package yqlib - -import ( - "bytes" - "encoding/base64" - "io" - - yaml "gopkg.in/yaml.v3" -) - -type base64Decoder struct { - reader io.Reader - finished bool - readAnything bool - encoding base64.Encoding -} - -func NewBase64Decoder() Decoder { - return &base64Decoder{finished: false, encoding: *base64.StdEncoding} -} - -func (dec *base64Decoder) Init(reader io.Reader) error { - dec.reader = reader - dec.readAnything = false - dec.finished = false - return nil -} - -func (dec *base64Decoder) Decode() (*CandidateNode, error) { - if dec.finished { - return nil, io.EOF - } - base64Reader := base64.NewDecoder(&dec.encoding, dec.reader) - buf := new(bytes.Buffer) - - if _, err := buf.ReadFrom(base64Reader); err != nil { - return nil, err - } - if buf.Len() == 0 { - dec.finished = true - - // if we've read _only_ an empty string, lets return that - // otherwise if we've already read some bytes, and now we get - // an empty string, then we are done. - if dec.readAnything { - return nil, io.EOF - } - } - dec.readAnything = true - return &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: buf.String(), - }, - }, nil -} diff --git a/external/yq/pkg/yqlib/decoder_csv_object.go b/external/yq/pkg/yqlib/decoder_csv_object.go deleted file mode 100755 index e8e7802..0000000 --- a/external/yq/pkg/yqlib/decoder_csv_object.go +++ /dev/null @@ -1,81 +0,0 @@ -package yqlib - -import ( - "encoding/csv" - "errors" - "io" - - "github.com/dimchansky/utfbom" - yaml "gopkg.in/yaml.v3" -) - -type csvObjectDecoder struct { - separator rune - reader csv.Reader - finished bool -} - -func NewCSVObjectDecoder(separator rune) Decoder { - return &csvObjectDecoder{separator: separator} -} - -func (dec *csvObjectDecoder) Init(reader io.Reader) error { - cleanReader, enc := utfbom.Skip(reader) - log.Debugf("Detected encoding: %s\n", enc) - dec.reader = *csv.NewReader(cleanReader) - dec.reader.Comma = dec.separator - dec.finished = false - return nil -} - -func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node { - node, err := parseSnippet(content) - if err != nil { - return createScalarNode(content, content) - } - return node -} - -func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node { - objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - - for i, header := range headerRow { - objectNode.Content = append( - objectNode.Content, - createScalarNode(header, header), - dec.convertToYamlNode(contentRow[i])) - } - return objectNode -} - -func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) { - if dec.finished { - return nil, io.EOF - } - headerRow, err := dec.reader.Read() - log.Debugf(": headerRow%v", headerRow) - if err != nil { - return nil, err - } - - rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - - contentRow, err := dec.reader.Read() - - for err == nil && len(contentRow) > 0 { - log.Debugf("Adding contentRow: %v", contentRow) - rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow)) - contentRow, err = dec.reader.Read() - log.Debugf("Read next contentRow: %v, %v", contentRow, err) - } - if !errors.Is(err, io.EOF) { - return nil, err - } - - return &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.DocumentNode, - Content: []*yaml.Node{rootArray}, - }, - }, nil -} diff --git a/external/yq/pkg/yqlib/decoder_json.go b/external/yq/pkg/yqlib/decoder_json.go deleted file mode 100755 index 448550a..0000000 --- a/external/yq/pkg/yqlib/decoder_json.go +++ /dev/null @@ -1,87 +0,0 @@ -package yqlib - -import ( - "fmt" - "io" - - "github.com/goccy/go-json" - yaml "gopkg.in/yaml.v3" -) - -type jsonDecoder struct { - decoder json.Decoder -} - -func NewJSONDecoder() Decoder { - return &jsonDecoder{} -} - -func (dec *jsonDecoder) Init(reader io.Reader) error { - dec.decoder = *json.NewDecoder(reader) - return nil -} - -func (dec *jsonDecoder) Decode() (*CandidateNode, error) { - - var dataBucket orderedMap - log.Debug("going to decode") - err := dec.decoder.Decode(&dataBucket) - if err != nil { - return nil, err - } - node, err := dec.convertToYamlNode(&dataBucket) - - if err != nil { - return nil, err - } - - return &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.DocumentNode, - Content: []*yaml.Node{node}, - }, - }, nil -} - -func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) { - if data.kv == nil { - switch rawData := data.altVal.(type) { - case nil: - return createScalarNode(nil, "null"), nil - case float64, float32: - // json decoder returns ints as float. - return parseSnippet(fmt.Sprintf("%v", rawData)) - case int, int64, int32, string, bool: - return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil - case []*orderedMap: - return dec.parseArray(rawData) - default: - return nil, fmt.Errorf("unrecognised type :( %v", rawData) - } - } - - var yamlMap = &yaml.Node{Kind: yaml.MappingNode} - for _, keyValuePair := range data.kv { - yamlValue, err := dec.convertToYamlNode(&keyValuePair.V) - if err != nil { - return nil, err - } - yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue) - } - return yamlMap, nil - -} - -func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) { - - var yamlMap = &yaml.Node{Kind: yaml.SequenceNode} - - for _, value := range dataArray { - yamlValue, err := dec.convertToYamlNode(value) - if err != nil { - return nil, err - } - yamlMap.Content = append(yamlMap.Content, yamlValue) - } - return yamlMap, nil -} diff --git a/external/yq/pkg/yqlib/decoder_properties.go b/external/yq/pkg/yqlib/decoder_properties.go deleted file mode 100755 index f234f9c..0000000 --- a/external/yq/pkg/yqlib/decoder_properties.go +++ /dev/null @@ -1,161 +0,0 @@ -package yqlib - -import ( - "bytes" - "fmt" - "io" - "strconv" - "strings" - - "github.com/magiconair/properties" - "gopkg.in/yaml.v3" -) - -type propertiesDecoder struct { - reader io.Reader - finished bool - d DataTreeNavigator -} - -func NewPropertiesDecoder() Decoder { - return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false} -} - -func (dec *propertiesDecoder) Init(reader io.Reader) error { - dec.reader = reader - dec.finished = false - return nil -} - -func parsePropKey(key string) []interface{} { - pathStrArray := strings.Split(key, ".") - path := make([]interface{}, len(pathStrArray)) - for i, pathStr := range pathStrArray { - num, err := strconv.ParseInt(pathStr, 10, 32) - if err == nil { - path[i] = num - } else { - path[i] = pathStr - } - } - return path -} - -func (dec *propertiesDecoder) processComment(c string) string { - if c == "" { - return "" - } - return "# " + c -} - -func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error { - assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} - - rhsCandidateNode := &CandidateNode{ - Path: path, - Node: &yaml.Node{ - Tag: "!!str", - Value: fmt.Sprintf("%v", path[len(path)-1]), - HeadComment: dec.processComment(strings.Join(comments, "\n")), - Kind: yaml.ScalarNode, - }, - } - - rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node) - - rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode} - - assignmentOpNode := &ExpressionNode{ - Operation: assignmentOp, - LHS: createTraversalTree(path, traversePreferences{}, true), - RHS: &ExpressionNode{Operation: rhsOp}, - } - - _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) - return err -} - -func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error { - value, _ := properties.Get(key) - path := parsePropKey(key) - - propertyComments := properties.GetComments(key) - if len(propertyComments) > 0 { - err := dec.applyPropertyComments(context, path, propertyComments) - if err != nil { - return nil - } - } - - rhsNode := &yaml.Node{ - Value: value, - Tag: "!!str", - Kind: yaml.ScalarNode, - } - - rhsNode.Tag = guessTagFromCustomType(rhsNode) - - rhsCandidateNode := &CandidateNode{ - Path: path, - Node: rhsNode, - } - - assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} - - rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode} - - assignmentOpNode := &ExpressionNode{ - Operation: assignmentOp, - LHS: createTraversalTree(path, traversePreferences{}, false), - RHS: &ExpressionNode{Operation: rhsOp}, - } - - _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) - return err -} - -func (dec *propertiesDecoder) Decode() (*CandidateNode, error) { - if dec.finished { - return nil, io.EOF - } - buf := new(bytes.Buffer) - - if _, err := buf.ReadFrom(dec.reader); err != nil { - return nil, err - } - if buf.Len() == 0 { - dec.finished = true - return nil, io.EOF - } - properties, err := properties.LoadString(buf.String()) - if err != nil { - return nil, err - } - properties.DisableExpansion = true - - rootMap := &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - }, - } - - context := Context{} - context = context.SingleChildContext(rootMap) - - for _, key := range properties.Keys() { - if err := dec.applyProperty(context, properties, key); err != nil { - return nil, err - } - - } - dec.finished = true - - return &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.DocumentNode, - Content: []*yaml.Node{rootMap.Node}, - }, - }, nil - -} diff --git a/external/yq/pkg/yqlib/decoder_test.go b/external/yq/pkg/yqlib/decoder_test.go deleted file mode 100755 index d3982b2..0000000 --- a/external/yq/pkg/yqlib/decoder_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "strings" -) - -type formatScenario struct { - input string - indent int - expression string - expected string - description string - subdescription string - skipDoc bool - scenarioType string - expectedError string -} - -func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) { - var output bytes.Buffer - writer := bufio.NewWriter(&output) - - if decoder == nil { - decoder = NewYamlDecoder(ConfiguredYamlPreferences) - } - - inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder) - if err != nil { - return "", err - } - - expression := s.expression - if expression == "" { - expression = "." - } - - exp, err := getExpressionParser().ParseExpression(expression) - - if err != nil { - return "", err - } - - context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp) - - if err != nil { - return "", err - } - - printer := NewPrinter(encoder, NewSinglePrinterWriter(writer)) - err = printer.PrintResults(context.MatchingNodes) - if err != nil { - return "", err - } - writer.Flush() - - return output.String(), nil -} - -func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string { - - result, err := processFormatScenario(s, decoder, encoder) - if err != nil { - panic(err) - } - return result - -} diff --git a/external/yq/pkg/yqlib/decoder_xml.go b/external/yq/pkg/yqlib/decoder_xml.go deleted file mode 100755 index ced6c96..0000000 --- a/external/yq/pkg/yqlib/decoder_xml.go +++ /dev/null @@ -1,348 +0,0 @@ -package yqlib - -import ( - "encoding/xml" - "errors" - "io" - "strings" - "unicode" - - "golang.org/x/net/html/charset" - yaml "gopkg.in/yaml.v3" -) - -type xmlDecoder struct { - reader io.Reader - readAnything bool - finished bool - prefs XmlPreferences -} - -func NewXMLDecoder(prefs XmlPreferences) Decoder { - return &xmlDecoder{ - finished: false, - prefs: prefs, - } -} - -func (dec *xmlDecoder) Init(reader io.Reader) error { - dec.reader = reader - dec.readAnything = false - dec.finished = false - return nil -} - -func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) { - yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for _, child := range nodes { - yamlChild, err := dec.convertToYamlNode(child) - if err != nil { - return nil, err - } - yamlNode.Content = append(yamlNode.Content, yamlChild) - } - - return yamlNode, nil -} - -func (dec *xmlDecoder) processComment(c string) string { - if c == "" { - return "" - } - return "#" + strings.TrimRight(c, " ") -} - -func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) { - log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment) - yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - - if len(n.Data) > 0 { - label := dec.prefs.ContentName - labelNode := createScalarNode(label, label) - labelNode.HeadComment = dec.processComment(n.HeadComment) - labelNode.FootComment = dec.processComment(n.FootComment) - yamlNode.Content = append(yamlNode.Content, labelNode, createScalarNode(n.Data, n.Data)) - } - - for i, keyValuePair := range n.Children { - label := keyValuePair.K - children := keyValuePair.V - labelNode := createScalarNode(label, label) - var valueNode *yaml.Node - var err error - - if i == 0 { - labelNode.HeadComment = dec.processComment(n.HeadComment) - - } - - labelNode.FootComment = dec.processComment(keyValuePair.FootComment) - - log.Debug("len of children in %v is %v", label, len(children)) - if len(children) > 1 { - valueNode, err = dec.createSequence(children) - if err != nil { - return nil, err - } - } else { - // comment hack for maps of scalars - // if the value is a scalar, the head comment of the scalar needs to go on the key? - // add tests for as well as multiple of inputXmlWithComments > yaml - if len(children[0].Children) == 0 && children[0].HeadComment != "" { - labelNode.HeadComment = labelNode.HeadComment + "\n" + strings.TrimSpace(children[0].HeadComment) - children[0].HeadComment = "" - } - valueNode, err = dec.convertToYamlNode(children[0]) - if err != nil { - return nil, err - } - } - yamlNode.Content = append(yamlNode.Content, labelNode, valueNode) - } - - return yamlNode, nil -} - -func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) { - if len(n.Children) > 0 { - return dec.createMap(n) - } - scalar := createScalarNode(n.Data, n.Data) - if n.Data == "" { - scalar = createScalarNode(nil, "") - } - log.Debug("scalar headC: %v, footC: %v", n.HeadComment, n.FootComment) - scalar.HeadComment = dec.processComment(n.HeadComment) - scalar.LineComment = dec.processComment(n.LineComment) - scalar.FootComment = dec.processComment(n.FootComment) - - return scalar, nil -} - -func (dec *xmlDecoder) Decode() (*CandidateNode, error) { - if dec.finished { - return nil, io.EOF - } - root := &xmlNode{} - // cant use xj - it doesn't keep map order. - err := dec.decodeXML(root) - - if err != nil { - return nil, err - } - firstNode, err := dec.convertToYamlNode(root) - - if err != nil { - return nil, err - } else if firstNode.Tag == "!!null" { - dec.finished = true - if dec.readAnything { - return nil, io.EOF - } - } - dec.readAnything = true - dec.finished = true - - return &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.DocumentNode, - Content: []*yaml.Node{firstNode}, - }, - }, nil -} - -type xmlNode struct { - Children []*xmlChildrenKv - HeadComment string - FootComment string - LineComment string - Data string -} - -type xmlChildrenKv struct { - K string - V []*xmlNode - FootComment string -} - -// AddChild appends a node to the list of children -func (n *xmlNode) AddChild(s string, c *xmlNode) { - - if n.Children == nil { - n.Children = make([]*xmlChildrenKv, 0) - } - log.Debug("looking for %s", s) - // see if we can find an existing entry to add to - for _, childEntry := range n.Children { - if childEntry.K == s { - log.Debug("found it, appending an entry%s", s) - childEntry.V = append(childEntry.V, c) - log.Debug("yay len of children in %v is %v", s, len(childEntry.V)) - return - } - } - log.Debug("not there, making a new one %s", s) - n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}}) -} - -type element struct { - parent *element - n *xmlNode - label string - state string -} - -// this code is heavily based on https://github.com/basgys/goxml2json -// main changes are to decode into a structure that preserves the original order -// of the map keys. -func (dec *xmlDecoder) decodeXML(root *xmlNode) error { - xmlDec := xml.NewDecoder(dec.reader) - xmlDec.Strict = dec.prefs.StrictMode - // That will convert the charset if the provided XML is non-UTF-8 - xmlDec.CharsetReader = charset.NewReaderLabel - - // Create first element from the root node - elem := &element{ - parent: nil, - n: root, - } - - getToken := func() (xml.Token, error) { - if dec.prefs.UseRawToken { - return xmlDec.RawToken() - } - return xmlDec.Token() - } - - for { - t, e := getToken() - if e != nil && !errors.Is(e, io.EOF) { - return e - } - if t == nil { - break - } - - switch se := t.(type) { - case xml.StartElement: - log.Debug("start element %v", se.Name.Local) - elem.state = "started" - // Build new a new current element and link it to its parent - elem = &element{ - parent: elem, - n: &xmlNode{}, - label: se.Name.Local, - } - - // Extract attributes as children - for _, a := range se.Attr { - if dec.prefs.KeepNamespace { - if a.Name.Space != "" { - a.Name.Local = a.Name.Space + ":" + a.Name.Local - } - } - elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: a.Value}) - } - case xml.CharData: - // Extract XML data (if any) - elem.n.Data = trimNonGraphic(string(se)) - if elem.n.Data != "" { - elem.state = "chardata" - log.Debug("chardata [%v] for %v", elem.n.Data, elem.label) - } - case xml.EndElement: - log.Debug("end element %v", elem.label) - elem.state = "finished" - // And add it to its parent list - if elem.parent != nil { - elem.parent.n.AddChild(elem.label, elem.n) - } - - // Then change the current element to its parent - elem = elem.parent - case xml.Comment: - - commentStr := string(xml.CharData(se)) - if elem.state == "started" { - applyFootComment(elem, commentStr) - - } else if elem.state == "chardata" { - log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) - elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr}) - } else { - log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) - elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr}) - } - - case xml.ProcInst: - if !dec.prefs.SkipProcInst { - elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: string(se.Inst)}) - } - case xml.Directive: - if !dec.prefs.SkipDirectives { - elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: string(se)}) - } - } - } - - return nil -} - -func applyFootComment(elem *element, commentStr string) { - - // first lets try to put the comment on the last child - if len(elem.n.Children) > 0 { - lastChildIndex := len(elem.n.Children) - 1 - childKv := elem.n.Children[lastChildIndex] - log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr) - childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr}) - } else { - log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr) - elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr}) - } -} - -func joinFilter(rawStrings []string) string { - stringsToJoin := make([]string, 0) - for _, str := range rawStrings { - if str != "" { - stringsToJoin = append(stringsToJoin, str) - } - } - return strings.Join(stringsToJoin, " ") -} - -// trimNonGraphic returns a slice of the string s, with all leading and trailing -// non graphic characters and spaces removed. -// -// Graphic characters include letters, marks, numbers, punctuation, symbols, -// and spaces, from categories L, M, N, P, S, Zs. -// Spacing characters are set by category Z and property Pattern_White_Space. -func trimNonGraphic(s string) string { - if s == "" { - return s - } - - var first *int - var last int - for i, r := range []rune(s) { - if !unicode.IsGraphic(r) || unicode.IsSpace(r) { - continue - } - - if first == nil { - f := i // copy i - first = &f - last = i - } else { - last = i - } - } - - // If first is nil, it means there are no graphic characters - if first == nil { - return "" - } - - return string([]rune(s)[*first : last+1]) -} diff --git a/external/yq/pkg/yqlib/decoder_yaml.go b/external/yq/pkg/yqlib/decoder_yaml.go deleted file mode 100755 index 207ec01..0000000 --- a/external/yq/pkg/yqlib/decoder_yaml.go +++ /dev/null @@ -1,114 +0,0 @@ -package yqlib - -import ( - "bufio" - "errors" - "io" - "regexp" - "strings" - - yaml "gopkg.in/yaml.v3" -) - -type yamlDecoder struct { - decoder yaml.Decoder - // work around of various parsing issues by yaml.v3 with document headers - prefs YamlPreferences - leadingContent string - readAnything bool - firstFile bool -} - -func NewYamlDecoder(prefs YamlPreferences) Decoder { - return &yamlDecoder{prefs: prefs, firstFile: true} -} - -func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) { - var commentLineRegEx = regexp.MustCompile(`^\s*#`) - var sb strings.Builder - for { - peekBytes, err := reader.Peek(3) - if errors.Is(err, io.EOF) { - // EOF are handled else where.. - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } else if string(peekBytes) == "---" { - _, err := reader.ReadString('\n') - sb.WriteString("$yqDocSeperator$\n") - if errors.Is(err, io.EOF) { - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } - } else if commentLineRegEx.MatchString(string(peekBytes)) { - line, err := reader.ReadString('\n') - sb.WriteString(line) - if errors.Is(err, io.EOF) { - return reader, sb.String(), nil - } else if err != nil { - return reader, sb.String(), err - } - } else { - return reader, sb.String(), nil - } - } -} - -func (dec *yamlDecoder) Init(reader io.Reader) error { - readerToUse := reader - leadingContent := "" - var err error - // if we 'evaluating together' - we only process the leading content - // of the first file - this ensures comments from subsequent files are - // merged together correctly. - if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) { - readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader)) - if err != nil { - return err - } - } - dec.leadingContent = leadingContent - dec.readAnything = false - dec.decoder = *yaml.NewDecoder(readerToUse) - dec.firstFile = false - return nil -} - -func (dec *yamlDecoder) Decode() (*CandidateNode, error) { - var dataBucket yaml.Node - - err := dec.decoder.Decode(&dataBucket) - if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { - // force returning an empty node with a comment. - dec.readAnything = true - return dec.blankNodeWithComment(), nil - - } else if err != nil { - return nil, err - } - - candidateNode := &CandidateNode{ - Node: &dataBucket, - } - - if dec.leadingContent != "" { - candidateNode.LeadingContent = dec.leadingContent - dec.leadingContent = "" - } - // move document comments into candidate node - // otherwise unwrap drops them. - candidateNode.TrailingContent = dataBucket.FootComment - dataBucket.FootComment = "" - return candidateNode, nil -} - -func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode { - return &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - LeadingContent: dec.leadingContent, - } -} diff --git a/external/yq/pkg/yqlib/doc/notification-snippet.md b/external/yq/pkg/yqlib/doc/notification-snippet.md deleted file mode 100755 index e69de29..0000000 diff --git a/external/yq/pkg/yqlib/doc/operators/add.md b/external/yq/pkg/yqlib/doc/operators/add.md deleted file mode 100755 index ecadb84..0000000 --- a/external/yq/pkg/yqlib/doc/operators/add.md +++ /dev/null @@ -1,319 +0,0 @@ -# Add - -Add behaves differently according to the type of the LHS: -* arrays: concatenate -* number scalars: arithmetic addition -* string scalars: concatenate -* maps: shallow merge (use the multiply operator (`*`) to deeply merge) - -Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. - - -## Concatenate arrays -Given a sample.yml file of: -```yaml -a: - - 1 - - 2 -b: - - 3 - - 4 -``` -then -```bash -yq '.a + .b' sample.yml -``` -will output -```yaml -- 1 -- 2 -- 3 -- 4 -``` - -## Concatenate to existing array -Note that the styling of `a` is kept. - -Given a sample.yml file of: -```yaml -a: [1,2] -b: - - 3 - - 4 -``` -then -```bash -yq '.a += .b' sample.yml -``` -will output -```yaml -a: [1, 2, 3, 4] -b: - - 3 - - 4 -``` - -## Concatenate null to array -Given a sample.yml file of: -```yaml -a: - - 1 - - 2 -``` -then -```bash -yq '.a + null' sample.yml -``` -will output -```yaml -- 1 -- 2 -``` - -## Append to existing array -Note that the styling is copied from existing array elements - -Given a sample.yml file of: -```yaml -a: ['dog'] -``` -then -```bash -yq '.a += "cat"' sample.yml -``` -will output -```yaml -a: ['dog', 'cat'] -``` - -## Add new object to array -Given a sample.yml file of: -```yaml -a: - - dog: woof -``` -then -```bash -yq '.a + {"cat": "meow"}' sample.yml -``` -will output -```yaml -- dog: woof -- cat: meow -``` - -## Relative append -Given a sample.yml file of: -```yaml -a: - a1: - b: - - cat - a2: - b: - - dog - a3: {} -``` -then -```bash -yq '.a[].b += ["mouse"]' sample.yml -``` -will output -```yaml -a: - a1: - b: - - cat - - mouse - a2: - b: - - dog - - mouse - a3: - b: - - mouse -``` - -## String concatenation -Given a sample.yml file of: -```yaml -a: cat -b: meow -``` -then -```bash -yq '.a += .b' sample.yml -``` -will output -```yaml -a: catmeow -b: meow -``` - -## Number addition - float -If the lhs or rhs are floats then the expression will be calculated with floats. - -Given a sample.yml file of: -```yaml -a: 3 -b: 4.9 -``` -then -```bash -yq '.a = .a + .b' sample.yml -``` -will output -```yaml -a: 7.9 -b: 4.9 -``` - -## Number addition - int -If both the lhs and rhs are ints then the expression will be calculated with ints. - -Given a sample.yml file of: -```yaml -a: 3 -b: 4 -``` -then -```bash -yq '.a = .a + .b' sample.yml -``` -will output -```yaml -a: 7 -b: 4 -``` - -## Increment numbers -Given a sample.yml file of: -```yaml -a: 3 -b: 5 -``` -then -```bash -yq '.[] += 1' sample.yml -``` -will output -```yaml -a: 4 -b: 6 -``` - -## Date addition -You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. - -Given a sample.yml file of: -```yaml -a: 2021-01-01T00:00:00Z -``` -then -```bash -yq '.a += "3h10m"' sample.yml -``` -will output -```yaml -a: 2021-01-01T03:10:00Z -``` - -## Date addition - custom format -You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` -then -```bash -yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 6:00AM GMT -``` - -## Add to null -Adding to null simply returns the rhs - -Running -```bash -yq --null-input 'null + "cat"' -``` -will output -```yaml -cat -``` - -## Add maps to shallow merge -Adding objects together shallow merges them. Use `*` to deeply merge. - -Given a sample.yml file of: -```yaml -a: - thing: - name: Astuff - value: x - a1: cool -b: - thing: - name: Bstuff - legs: 3 - b1: neat -``` -then -```bash -yq '.a += .b' sample.yml -``` -will output -```yaml -a: - thing: - name: Bstuff - legs: 3 - a1: cool - b1: neat -b: - thing: - name: Bstuff - legs: 3 - b1: neat -``` - -## Custom types: that are really strings -When custom tags are encountered, yq will try to decode the underlying type. - -Given a sample.yml file of: -```yaml -a: !horse cat -b: !goat _meow -``` -then -```bash -yq '.a += .b' sample.yml -``` -will output -```yaml -a: !horse cat_meow -b: !goat _meow -``` - -## Custom types: that are really numbers -When custom tags are encountered, yq will try to decode the underlying type. - -Given a sample.yml file of: -```yaml -a: !horse 1.2 -b: !goat 2.3 -``` -then -```bash -yq '.a += .b' sample.yml -``` -will output -```yaml -a: !horse 3.5 -b: !goat 2.3 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/alternative-default-value.md b/external/yq/pkg/yqlib/doc/operators/alternative-default-value.md deleted file mode 100755 index 127465e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/alternative-default-value.md +++ /dev/null @@ -1,108 +0,0 @@ -# Alternative (Default value) - -This operator is used to provide alternative (or default) values when a particular expression is either null or false. - -## LHS is defined -Given a sample.yml file of: -```yaml -a: bridge -``` -then -```bash -yq '.a // "hello"' sample.yml -``` -will output -```yaml -bridge -``` - -## LHS is not defined -Given a sample.yml file of: -```yaml -{} -``` -then -```bash -yq '.a // "hello"' sample.yml -``` -will output -```yaml -hello -``` - -## LHS is null -Given a sample.yml file of: -```yaml -a: ~ -``` -then -```bash -yq '.a // "hello"' sample.yml -``` -will output -```yaml -hello -``` - -## LHS is false -Given a sample.yml file of: -```yaml -a: false -``` -then -```bash -yq '.a // "hello"' sample.yml -``` -will output -```yaml -hello -``` - -## RHS is an expression -Given a sample.yml file of: -```yaml -a: false -b: cat -``` -then -```bash -yq '.a // .b' sample.yml -``` -will output -```yaml -cat -``` - -## Update or create - entity exists -This initialises `a` if it's not present - -Given a sample.yml file of: -```yaml -a: 1 -``` -then -```bash -yq '(.a // (.a = 0)) += 1' sample.yml -``` -will output -```yaml -a: 2 -``` - -## Update or create - entity does not exist -This initialises `a` if it's not present - -Given a sample.yml file of: -```yaml -b: camel -``` -then -```bash -yq '(.a // (.a = 0)) += 1' sample.yml -``` -will output -```yaml -b: camel -a: 1 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/anchor-and-alias-operators.md b/external/yq/pkg/yqlib/doc/operators/anchor-and-alias-operators.md deleted file mode 100755 index 8dbfd99..0000000 --- a/external/yq/pkg/yqlib/doc/operators/anchor-and-alias-operators.md +++ /dev/null @@ -1,333 +0,0 @@ -# Anchor and Alias Operators - -Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names). - -`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. - - -## Merge one map -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: *CENTER - r: 10 -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -x: 1 -y: 2 -r: 10 -``` - -## Merge multiple maps -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *CENTER - - *BIG -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - -## Override -see https://yaml.org/type/merge.html - -Given a sample.yml file of: -```yaml -- &CENTER - x: 1 - y: 2 -- &LEFT - x: 0 - y: 2 -- &BIG - r: 10 -- &SMALL - r: 1 -- !!merge <<: - - *BIG - - *LEFT - - *SMALL - x: 1 -``` -then -```bash -yq '.[4] | explode(.)' sample.yml -``` -will output -```yaml -r: 10 -x: 1 -y: 2 -``` - -## Get anchor -Given a sample.yml file of: -```yaml -a: &billyBob cat -``` -then -```bash -yq '.a | anchor' sample.yml -``` -will output -```yaml -billyBob -``` - -## Set anchor -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a anchor = "foobar"' sample.yml -``` -will output -```yaml -a: &foobar cat -``` - -## Set anchor relatively using assign-update -Given a sample.yml file of: -```yaml -a: - b: cat -``` -then -```bash -yq '.a anchor |= .b' sample.yml -``` -will output -```yaml -a: &cat - b: cat -``` - -## Get alias -Given a sample.yml file of: -```yaml -b: &billyBob meow -a: *billyBob -``` -then -```bash -yq '.a | alias' sample.yml -``` -will output -```yaml -billyBob -``` - -## Set alias -Given a sample.yml file of: -```yaml -b: &meow purr -a: cat -``` -then -```bash -yq '.a alias = "meow"' sample.yml -``` -will output -```yaml -b: &meow purr -a: *meow -``` - -## Set alias to blank does nothing -Given a sample.yml file of: -```yaml -b: &meow purr -a: cat -``` -then -```bash -yq '.a alias = ""' sample.yml -``` -will output -```yaml -b: &meow purr -a: cat -``` - -## Set alias relatively using assign-update -Given a sample.yml file of: -```yaml -b: &meow purr -a: - f: meow -``` -then -```bash -yq '.a alias |= .f' sample.yml -``` -will output -```yaml -b: &meow purr -a: *meow -``` - -## Explode alias and anchor -Given a sample.yml file of: -```yaml -f: - a: &a cat - b: *a -``` -then -```bash -yq 'explode(.f)' sample.yml -``` -will output -```yaml -f: - a: cat - b: cat -``` - -## Explode with no aliases or anchors -Given a sample.yml file of: -```yaml -a: mike -``` -then -```bash -yq 'explode(.a)' sample.yml -``` -will output -```yaml -a: mike -``` - -## Explode with alias keys -Given a sample.yml file of: -```yaml -f: - a: &a cat - *a: b -``` -then -```bash -yq 'explode(.f)' sample.yml -``` -will output -```yaml -f: - a: cat - cat: b -``` - -## Explode with merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq 'explode(.)' sample.yml -``` -will output -```yaml -foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: bar_b - thing: foo_thing - c: foobarList_c - a: foo_a -foobar: - c: foo_c - a: foo_a - thing: foobar_thing -``` - -## Dereference and update a field -`Use explode with multiply to dereference an object - -Given a sample.yml file of: -```yaml -item_value: &item_value - value: true -thingOne: - name: item_1 - !!merge <<: *item_value -thingTwo: - name: item_2 - !!merge <<: *item_value -``` -then -```bash -yq '.thingOne |= explode(.) * {"value": false}' sample.yml -``` -will output -```yaml -item_value: &item_value - value: true -thingOne: - name: item_1 - value: false -thingTwo: - name: item_2 - !!merge <<: *item_value -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/array-to-map.md b/external/yq/pkg/yqlib/doc/operators/array-to-map.md deleted file mode 100755 index 5930aaa..0000000 --- a/external/yq/pkg/yqlib/doc/operators/array-to-map.md +++ /dev/null @@ -1,28 +0,0 @@ -# Array to Map - -Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over. - -Behind the scenes, this is implemented using reduce: - -``` -(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i) -``` - -## Simple example -Given a sample.yml file of: -```yaml -cool: - - null - - null - - hello -``` -then -```bash -yq '.cool |= array_to_map' sample.yml -``` -will output -```yaml -cool: - 2: hello -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/assign-update.md b/external/yq/pkg/yqlib/doc/operators/assign-update.md deleted file mode 100755 index 5885efc..0000000 --- a/external/yq/pkg/yqlib/doc/operators/assign-update.md +++ /dev/null @@ -1,270 +0,0 @@ -# Assign (Update) - -This operator is used to update node values. It can be used in either the: - -### plain form: `=` -Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline. - -### relative form: `|=` -This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment. - - -### Flags -- `c` clobber custom tags - -## Create yaml file -Running -```bash -yq --null-input '.a.b = "cat" | .x = "frog"' -``` -will output -```yaml -a: - b: cat -x: frog -``` - -## Update node to be the child value -Given a sample.yml file of: -```yaml -a: - b: - g: foof -``` -then -```bash -yq '.a |= .b' sample.yml -``` -will output -```yaml -a: - g: foof -``` - -## Double elements in an array -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq '.[] |= . * 2' sample.yml -``` -will output -```yaml -- 2 -- 4 -- 6 -``` - -## Update node from another file -Note this will also work when the second file is a scalar (string/number) - -Given a sample.yml file of: -```yaml -a: apples -``` -And another sample another.yml file of: -```yaml -b: bob -``` -then -```bash -yq eval-all 'select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)' sample.yml another.yml -``` -will output -```yaml -a: - b: bob -``` - -## Update node to be the sibling value -Given a sample.yml file of: -```yaml -a: - b: child -b: sibling -``` -then -```bash -yq '.a = .b' sample.yml -``` -will output -```yaml -a: sibling -b: sibling -``` - -## Updated multiple paths -Given a sample.yml file of: -```yaml -a: fieldA -b: fieldB -c: fieldC -``` -then -```bash -yq '(.a, .c) = "potato"' sample.yml -``` -will output -```yaml -a: potato -b: fieldB -c: potato -``` - -## Update string value -Given a sample.yml file of: -```yaml -a: - b: apple -``` -then -```bash -yq '.a.b = "frog"' sample.yml -``` -will output -```yaml -a: - b: frog -``` - -## Update string value via |= -Note there is no difference between `=` and `|=` when the RHS is a scalar - -Given a sample.yml file of: -```yaml -a: - b: apple -``` -then -```bash -yq '.a.b |= "frog"' sample.yml -``` -will output -```yaml -a: - b: frog -``` - -## Update deeply selected results -Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet. - -Given a sample.yml file of: -```yaml -a: - b: apple - c: cactus -``` -then -```bash -yq '(.a[] | select(. == "apple")) = "frog"' sample.yml -``` -will output -```yaml -a: - b: frog - c: cactus -``` - -## Update array values -Given a sample.yml file of: -```yaml -- candy -- apple -- sandy -``` -then -```bash -yq '(.[] | select(. == "*andy")) = "bogs"' sample.yml -``` -will output -```yaml -- bogs -- apple -- bogs -``` - -## Update empty object -Given a sample.yml file of: -```yaml -{} -``` -then -```bash -yq '.a.b |= "bogs"' sample.yml -``` -will output -```yaml -a: - b: bogs -``` - -## Update node value that has an anchor -Anchor will remaple - -Given a sample.yml file of: -```yaml -a: &cool cat -``` -then -```bash -yq '.a = "dog"' sample.yml -``` -will output -```yaml -a: &cool dog -``` - -## Update empty object and array -Given a sample.yml file of: -```yaml -{} -``` -then -```bash -yq '.a.b.[0] |= "bogs"' sample.yml -``` -will output -```yaml -a: - b: - - bogs -``` - -## Custom types are maintained by default -Given a sample.yml file of: -```yaml -a: !cat meow -b: !dog woof -``` -then -```bash -yq '.a = .b' sample.yml -``` -will output -```yaml -a: !cat woof -b: !dog woof -``` - -## Custom types: clovver -Use the `c` option to clobber custom tags - -Given a sample.yml file of: -```yaml -a: !cat meow -b: !dog woof -``` -then -```bash -yq '.a =c .b' sample.yml -``` -will output -```yaml -a: !dog woof -b: !dog woof -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/boolean-operators.md b/external/yq/pkg/yqlib/doc/operators/boolean-operators.md deleted file mode 100755 index c409445..0000000 --- a/external/yq/pkg/yqlib/doc/operators/boolean-operators.md +++ /dev/null @@ -1,228 +0,0 @@ -# Boolean Operators - -The `or` and `and` operators take two parameters and return a boolean result. - -`not` flips a boolean from true to false, or vice versa. - -`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true. - -`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet - -These are most commonly used with the `select` operator to filter particular nodes. - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) - -## `or` example -Running -```bash -yq --null-input 'true or false' -``` -will output -```yaml -true -``` - -## `and` example -Running -```bash -yq --null-input 'true and false' -``` -will output -```yaml -false -``` - -## Matching nodes with select, equals and or -Given a sample.yml file of: -```yaml -- a: bird - b: dog -- a: frog - b: bird -- a: cat - b: fly -``` -then -```bash -yq '[.[] | select(.a == "cat" or .b == "dog")]' sample.yml -``` -will output -```yaml -- a: bird - b: dog -- a: cat - b: fly -``` - -## `any` returns true if any boolean in a given array is true -Given a sample.yml file of: -```yaml -- false -- true -``` -then -```bash -yq 'any' sample.yml -``` -will output -```yaml -true -``` - -## `any` returns false for an empty array -Given a sample.yml file of: -```yaml -[] -``` -then -```bash -yq 'any' sample.yml -``` -will output -```yaml -false -``` - -## `any_c` returns true if any element in the array is true for the given condition. -Given a sample.yml file of: -```yaml -a: - - rad - - awesome -b: - - meh - - whatever -``` -then -```bash -yq '.[] |= any_c(. == "awesome")' sample.yml -``` -will output -```yaml -a: true -b: false -``` - -## `all` returns true if all booleans in a given array are true -Given a sample.yml file of: -```yaml -- true -- true -``` -then -```bash -yq 'all' sample.yml -``` -will output -```yaml -true -``` - -## `all` returns true for an empty array -Given a sample.yml file of: -```yaml -[] -``` -then -```bash -yq 'all' sample.yml -``` -will output -```yaml -true -``` - -## `all_c` returns true if all elements in the array are true for the given condition. -Given a sample.yml file of: -```yaml -a: - - rad - - awesome -b: - - meh - - 12 -``` -then -```bash -yq '.[] |= all_c(tag == "!!str")' sample.yml -``` -will output -```yaml -a: true -b: false -``` - -## Not true is false -Running -```bash -yq --null-input 'true | not' -``` -will output -```yaml -false -``` - -## Not false is true -Running -```bash -yq --null-input 'false | not' -``` -will output -```yaml -true -``` - -## String values considered to be true -Running -```bash -yq --null-input '"cat" | not' -``` -will output -```yaml -false -``` - -## Empty string value considered to be true -Running -```bash -yq --null-input '"" | not' -``` -will output -```yaml -false -``` - -## Numbers are considered to be true -Running -```bash -yq --null-input '1 | not' -``` -will output -```yaml -false -``` - -## Zero is considered to be true -Running -```bash -yq --null-input '0 | not' -``` -will output -```yaml -false -``` - -## Null is considered to be false -Running -```bash -yq --null-input '~ | not' -``` -will output -```yaml -true -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/collect-into-array.md b/external/yq/pkg/yqlib/doc/operators/collect-into-array.md deleted file mode 100755 index 73152e5..0000000 --- a/external/yq/pkg/yqlib/doc/operators/collect-into-array.md +++ /dev/null @@ -1,41 +0,0 @@ -# Collect into Array - -This creates an array using the expression between the square brackets. - - -## Collect empty -Running -```bash -yq --null-input '[]' -``` -will output -```yaml -[] -``` - -## Collect single -Running -```bash -yq --null-input '["cat"]' -``` -will output -```yaml -- cat -``` - -## Collect many -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq '[.a, .b]' sample.yml -``` -will output -```yaml -- cat -- dog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/column.md b/external/yq/pkg/yqlib/doc/operators/column.md deleted file mode 100755 index d2cef94..0000000 --- a/external/yq/pkg/yqlib/doc/operators/column.md +++ /dev/null @@ -1,60 +0,0 @@ -# Column - -Returns the column of the matching node. Starts from 1, 0 indicates there was no column data. - -## Returns column of _value_ node -Given a sample.yml file of: -```yaml -a: cat -b: bob -``` -then -```bash -yq '.b | column' sample.yml -``` -will output -```yaml -4 -``` - -## Returns column of _key_ node -Pipe through the key operator to get the column of the key - -Given a sample.yml file of: -```yaml -a: cat -b: bob -``` -then -```bash -yq '.b | key | column' sample.yml -``` -will output -```yaml -1 -``` - -## First column is 1 -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a | key | column' sample.yml -``` -will output -```yaml -1 -``` - -## No column data is 0 -Running -```bash -yq --null-input '{"a": "new entry"} | column' -``` -will output -```yaml -0 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/comment-operators.md b/external/yq/pkg/yqlib/doc/operators/comment-operators.md deleted file mode 100755 index 9d1b9da..0000000 --- a/external/yq/pkg/yqlib/doc/operators/comment-operators.md +++ /dev/null @@ -1,337 +0,0 @@ -# Comment Operators - -Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples. - -Like the `=` and `|=` assign operators, the same syntax applies when updating comments: - -### plain form: `=` -This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline - -### relative form: `|=` -Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path. - -## Set line comment -Set the comment on the key node for more reliability (see below). - -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a line_comment="single"' sample.yml -``` -will output -```yaml -a: cat # single -``` - -## Set line comment of a maps/arrays -For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars. - -Given a sample.yml file of: -```yaml -a: - b: things -``` -then -```bash -yq '(.a | key) line_comment="single"' sample.yml -``` -will output -```yaml -a: # single - b: things -``` - -## Use update assign to perform relative updates -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq '.. line_comment |= .' sample.yml -``` -will output -```yaml -a: cat # cat -b: dog # dog -``` - -## Where is the comment - map key example -The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). -From this, you can see the 'hello-world-comment' is actually on the 'hello' key - -Given a sample.yml file of: -```yaml -hello: # hello-world-comment - message: world -``` -then -```bash -yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml -``` -will output -```yaml -- p: "" - isKey: false - hc: "" - lc: "" - fc: "" -- p: hello - isKey: true - hc: "" - lc: hello-world-comment - fc: "" -- p: hello - isKey: false - hc: "" - lc: "" - fc: "" -- p: hello.message - isKey: true - hc: "" - lc: "" - fc: "" -- p: hello.message - isKey: false - hc: "" - lc: "" - fc: "" -``` - -## Retrieve comment - map key example -From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment - -Given a sample.yml file of: -```yaml -hello: # hello-world-comment - message: world -``` -then -```bash -yq '.hello | key | line_comment' sample.yml -``` -will output -```yaml -hello-world-comment -``` - -## Where is the comment - array example -The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). -From this, you can see the 'under-name-comment' is actually on the first child - -Given a sample.yml file of: -```yaml -name: - # under-name-comment - - first-array-child -``` -then -```bash -yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml -``` -will output -```yaml -- p: "" - isKey: false - hc: "" - lc: "" - fc: "" -- p: name - isKey: true - hc: "" - lc: "" - fc: "" -- p: name - isKey: false - hc: "" - lc: "" - fc: "" -- p: name.0 - isKey: false - hc: under-name-comment - lc: "" - fc: "" -``` - -## Retrieve comment - array example -From the previous example, we know that the comment is on the first child as a headComment - -Given a sample.yml file of: -```yaml -name: - # under-name-comment - - first-array-child -``` -then -```bash -yq '.name[0] | headComment' sample.yml -``` -will output -```yaml -under-name-comment -``` - -## Set head comment -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '. head_comment="single"' sample.yml -``` -will output -```yaml -# single - -a: cat -``` - -## Set head comment of a map entry -Given a sample.yml file of: -```yaml -f: foo -a: - b: cat -``` -then -```bash -yq '(.a | key) head_comment="single"' sample.yml -``` -will output -```yaml -f: foo -# single -a: - b: cat -``` - -## Set foot comment, using an expression -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '. foot_comment=.a' sample.yml -``` -will output -```yaml -a: cat -# cat -``` - -## Remove comment -Given a sample.yml file of: -```yaml -a: cat # comment -b: dog # leave this -``` -then -```bash -yq '.a line_comment=""' sample.yml -``` -will output -```yaml -a: cat -b: dog # leave this -``` - -## Remove (strip) all comments -Note the use of `...` to ensure key nodes are included. - -Given a sample.yml file of: -```yaml -# hi -a: cat # comment -# great -b: # key comment -``` -then -```bash -yq '... comments=""' sample.yml -``` -will output -```yaml -a: cat -b: -``` - -## Get line comment -Given a sample.yml file of: -```yaml -# welcome! -a: cat # meow -# have a great day -``` -then -```bash -yq '.a | line_comment' sample.yml -``` -will output -```yaml -meow -``` - -## Get head comment -Given a sample.yml file of: -```yaml -# welcome! - -a: cat # meow - -# have a great day -``` -then -```bash -yq '. | head_comment' sample.yml -``` -will output -```yaml -welcome! -``` - -## Head comment with document split -Given a sample.yml file of: -```yaml -# welcome! ---- -# bob -a: cat # meow - -# have a great day -``` -then -```bash -yq 'head_comment' sample.yml -``` -will output -```yaml -welcome! -bob -``` - -## Get foot comment -Given a sample.yml file of: -```yaml -# welcome! - -a: cat # meow - -# have a great day -# no really -``` -then -```bash -yq '. | foot_comment' sample.yml -``` -will output -```yaml -have a great day -no really -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/compare.md b/external/yq/pkg/yqlib/doc/operators/compare.md deleted file mode 100755 index fbd1b1e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/compare.md +++ /dev/null @@ -1,100 +0,0 @@ -# Compare Operators - -Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. - -The following types are currently supported: - -- numbers -- strings -- datetimes - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) - -## Compare numbers (>) -Given a sample.yml file of: -```yaml -a: 5 -b: 4 -``` -then -```bash -yq '.a > .b' sample.yml -``` -will output -```yaml -true -``` - -## Compare equal numbers (>=) -Given a sample.yml file of: -```yaml -a: 5 -b: 5 -``` -then -```bash -yq '.a >= .b' sample.yml -``` -will output -```yaml -true -``` - -## Compare strings -Compares strings by their bytecode. - -Given a sample.yml file of: -```yaml -a: zoo -b: apple -``` -then -```bash -yq '.a > .b' sample.yml -``` -will output -```yaml -true -``` - -## Compare date times -You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. - -Given a sample.yml file of: -```yaml -a: 2021-01-01T03:10:00Z -b: 2020-01-01T03:10:00Z -``` -then -```bash -yq '.a > .b' sample.yml -``` -will output -```yaml -true -``` - -## Both sides are null: > is false -Running -```bash -yq --null-input '.a > .b' -``` -will output -```yaml -false -``` - -## Both sides are null: >= is true -Running -```bash -yq --null-input '.a >= .b' -``` -will output -```yaml -true -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/contains.md b/external/yq/pkg/yqlib/doc/operators/contains.md deleted file mode 100755 index dfdd5a0..0000000 --- a/external/yq/pkg/yqlib/doc/operators/contains.md +++ /dev/null @@ -1,114 +0,0 @@ -# Contains - -This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring. - -{% hint style="warning" %} - -_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`. - -See the "Array has a subset array" example below on how to check for a subset. - -{% endhint %} - -## Array contains array -Array is equal or subset of - -Given a sample.yml file of: -```yaml -- foobar -- foobaz -- blarp -``` -then -```bash -yq 'contains(["baz", "bar"])' sample.yml -``` -will output -```yaml -true -``` - -## Array has a subset array -Subtract the superset array from the subset, if there's anything left, it's not a subset - -Given a sample.yml file of: -```yaml -- foobar -- foobaz -- blarp -``` -then -```bash -yq '["baz", "bar"] - . | length == 0' sample.yml -``` -will output -```yaml -false -``` - -## Object included in array -Given a sample.yml file of: -```yaml -"foo": 12 -"bar": - - 1 - - 2 - - "barp": 12 - "blip": 13 -``` -then -```bash -yq 'contains({"bar": [{"barp": 12}]})' sample.yml -``` -will output -```yaml -true -``` - -## Object not included in array -Given a sample.yml file of: -```yaml -"foo": 12 -"bar": - - 1 - - 2 - - "barp": 12 - "blip": 13 -``` -then -```bash -yq 'contains({"foo": 12, "bar": [{"barp": 15}]})' sample.yml -``` -will output -```yaml -false -``` - -## String contains substring -Given a sample.yml file of: -```yaml -foobar -``` -then -```bash -yq 'contains("bar")' sample.yml -``` -will output -```yaml -true -``` - -## String equals string -Given a sample.yml file of: -```yaml -meow -``` -then -```bash -yq 'contains("meow")' sample.yml -``` -will output -```yaml -true -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/create-collect-into-object.md b/external/yq/pkg/yqlib/doc/operators/create-collect-into-object.md deleted file mode 100755 index 2c132bb..0000000 --- a/external/yq/pkg/yqlib/doc/operators/create-collect-into-object.md +++ /dev/null @@ -1,95 +0,0 @@ -# Create, Collect into Object - -This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents. - -## Collect empty object -Running -```bash -yq --null-input '{}' -``` -will output -```yaml -{} -``` - -## Wrap (prefix) existing object -Given a sample.yml file of: -```yaml -name: Mike -``` -then -```bash -yq '{"wrap": .}' sample.yml -``` -will output -```yaml -wrap: - name: Mike -``` - -## Using splat to create multiple objects -Given a sample.yml file of: -```yaml -name: Mike -pets: - - cat - - dog -``` -then -```bash -yq '{.name: .pets.[]}' sample.yml -``` -will output -```yaml -Mike: cat -Mike: dog -``` - -## Working with multiple documents -Given a sample.yml file of: -```yaml -name: Mike -pets: - - cat - - dog ---- -name: Rosey -pets: - - monkey - - sheep -``` -then -```bash -yq '{.name: .pets.[]}' sample.yml -``` -will output -```yaml -Mike: cat -Mike: dog -Rosey: monkey -Rosey: sheep -``` - -## Creating yaml from scratch -Running -```bash -yq --null-input '{"wrap": "frog"}' -``` -will output -```yaml -wrap: frog -``` - -## Creating yaml from scratch with multiple objects -Running -```bash -yq --null-input '(.a.b = "foo") | (.d.e = "bar")' -``` -will output -```yaml -a: - b: foo -d: - e: bar -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/datetime.md b/external/yq/pkg/yqlib/doc/operators/datetime.md deleted file mode 100755 index 0dc3ed5..0000000 --- a/external/yq/pkg/yqlib/doc/operators/datetime.md +++ /dev/null @@ -1,197 +0,0 @@ -# Date Time - -Various operators for parsing and manipulating dates. - -## Date time formattings -This uses the golangs built in time library for parsing and formatting date times. - -When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. - -To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. - -```bash -yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' -``` - -See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options. - - -## Timezones -This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details. - - -## Durations -Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. - -You can durations to time using the `+` operator. - -## Format: from standard RFC3339 format -Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node. - -Given a sample.yml file of: -```yaml -a: 2001-12-15T02:59:43.1Z -``` -then -```bash -yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 2:59AM -``` - -## Format: from custom date time -Use with_dtf to set a custom datetime format for parsing. - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM -``` -then -```bash -yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml -``` -will output -```yaml -a: 2001-12-15 -``` - -## Format: get the day of the week -Given a sample.yml file of: -```yaml -a: 2001-12-15 -``` -then -```bash -yq '.a | format_datetime("Monday")' sample.yml -``` -will output -```yaml -Saturday -``` - -## Now -Given a sample.yml file of: -```yaml -a: cool -``` -then -```bash -yq '.updated = now' sample.yml -``` -will output -```yaml -a: cool -updated: 2021-05-19T01:02:03Z -``` - -## Timezone: from standard RFC3339 format -Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format. - -Given a sample.yml file of: -```yaml -a: cool -``` -then -```bash -yq '.updated = (now | tz("Australia/Sydney"))' sample.yml -``` -will output -```yaml -a: cool -updated: 2021-05-19T11:02:03+10:00 -``` - -## Timezone: with custom format -Specify standard IANA Time Zone format or 'utc', 'local' - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` -then -```bash -yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 1:59PM AEDT -``` - -## Add and tz custom format -Specify standard IANA Time Zone format or 'utc', 'local' - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` -then -```bash -yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 1:59PM AEDT -``` - -## Date addition -Given a sample.yml file of: -```yaml -a: 2021-01-01T00:00:00Z -``` -then -```bash -yq '.a += "3h10m"' sample.yml -``` -will output -```yaml -a: 2021-01-01T03:10:00Z -``` - -## Date subtraction -You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information. - -Given a sample.yml file of: -```yaml -a: 2021-01-01T03:10:00Z -``` -then -```bash -yq '.a -= "3h10m"' sample.yml -``` -will output -```yaml -a: 2021-01-01T00:00:00Z -``` - -## Date addition - custom format -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` -then -```bash -yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 6:00AM GMT -``` - -## Date script with custom format -You can embed full expressions in with_dtf if needed. - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` -then -```bash -yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 2:00PM AWST -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/delete.md b/external/yq/pkg/yqlib/doc/operators/delete.md deleted file mode 100755 index 6d0a99a..0000000 --- a/external/yq/pkg/yqlib/doc/operators/delete.md +++ /dev/null @@ -1,120 +0,0 @@ -# Delete - -Deletes matching entries in maps or arrays. - -## Delete entry in map -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq 'del(.b)' sample.yml -``` -will output -```yaml -a: cat -``` - -## Delete nested entry in map -Given a sample.yml file of: -```yaml -a: - a1: fred - a2: frood -``` -then -```bash -yq 'del(.a.a1)' sample.yml -``` -will output -```yaml -a: - a2: frood -``` - -## Delete entry in array -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq 'del(.[1])' sample.yml -``` -will output -```yaml -- 1 -- 3 -``` - -## Delete nested entry in array -Given a sample.yml file of: -```yaml -- a: cat - b: dog -``` -then -```bash -yq 'del(.[0].a)' sample.yml -``` -will output -```yaml -- b: dog -``` - -## Delete no matches -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq 'del(.c)' sample.yml -``` -will output -```yaml -a: cat -b: dog -``` - -## Delete matching entries -Given a sample.yml file of: -```yaml -a: cat -b: dog -c: bat -``` -then -```bash -yq 'del( .[] | select(. == "*at") )' sample.yml -``` -will output -```yaml -b: dog -``` - -## Recursively delete matching keys -Given a sample.yml file of: -```yaml -a: - name: frog - b: - name: blog - age: 12 -``` -then -```bash -yq 'del(.. | select(has("name")).name)' sample.yml -``` -will output -```yaml -a: - b: - age: 12 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/document-index.md b/external/yq/pkg/yqlib/doc/operators/document-index.md deleted file mode 100755 index 619fd39..0000000 --- a/external/yq/pkg/yqlib/doc/operators/document-index.md +++ /dev/null @@ -1,91 +0,0 @@ -# Document Index - -Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document. - -## Retrieve a document index -Given a sample.yml file of: -```yaml -a: cat ---- -a: frog -``` -then -```bash -yq '.a | document_index' sample.yml -``` -will output -```yaml -0 ---- -1 -``` - -## Retrieve a document index, shorthand -Given a sample.yml file of: -```yaml -a: cat ---- -a: frog -``` -then -```bash -yq '.a | di' sample.yml -``` -will output -```yaml -0 ---- -1 -``` - -## Filter by document index -Given a sample.yml file of: -```yaml -a: cat ---- -a: frog -``` -then -```bash -yq 'select(document_index == 1)' sample.yml -``` -will output -```yaml -a: frog -``` - -## Filter by document index shorthand -Given a sample.yml file of: -```yaml -a: cat ---- -a: frog -``` -then -```bash -yq 'select(di == 1)' sample.yml -``` -will output -```yaml -a: frog -``` - -## Print Document Index with matches -Given a sample.yml file of: -```yaml -a: cat ---- -a: frog -``` -then -```bash -yq '.a | ({"match": ., "doc": document_index})' sample.yml -``` -will output -```yaml -match: cat -doc: 0 -match: frog -doc: 1 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/encode-decode.md b/external/yq/pkg/yqlib/doc/operators/encode-decode.md deleted file mode 100755 index f9e5097..0000000 --- a/external/yq/pkg/yqlib/doc/operators/encode-decode.md +++ /dev/null @@ -1,470 +0,0 @@ -# Encoder / Decoder - -Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. - -Note that you can optionally pass an indent value to the encode functions (see below). - -These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them. - - -| Format | Decode (from string) | Encode (to string) | -| --- | -- | --| -| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml | -| JSON | from_json/@jsond | to_json(i)/@json | -| Properties | from_props/@propsd | to_props/@props | -| CSV | from_csv/@csvd | to_csv/@csv | -| TSV | from_tsv/@tsvd | to_tsv/@tsv | -| XML | from_xml/@xmld | to_xml(i)/@xml | -| Base64 | @base64d | @base64 | - - -See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. - -XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. - - -Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string. - -## Encode value as json string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq '.b = (.a | to_json)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - { - "cool": "thing" - } -``` - -## Encode value as json string, on one line -Pass in a 0 indent to print json on a single line. - -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq '.b = (.a | to_json(0))' sample.yml -``` -will output -```yaml -a: - cool: thing -b: '{"cool":"thing"}' -``` - -## Encode value as json string, on one line shorthand -Pass in a 0 indent to print json on a single line. - -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq '.b = (.a | @json)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: '{"cool":"thing"}' -``` - -## Decode a json encoded string -Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling. - -Given a sample.yml file of: -```yaml -a: '{"cool":"thing"}' -``` -then -```bash -yq '.a | from_json | ... style=""' sample.yml -``` -will output -```yaml -cool: thing -``` - -## Encode value as props string -Given a sample.yml file of: -```yaml -a: - cool: thing -``` -then -```bash -yq '.b = (.a | @props)' sample.yml -``` -will output -```yaml -a: - cool: thing -b: | - cool = thing -``` - -## Decode props encoded string -Given a sample.yml file of: -```yaml -a: |- - cats=great - dogs=cool as well -``` -then -```bash -yq '.a |= @propsd' sample.yml -``` -will output -```yaml -a: - cats: great - dogs: cool as well -``` - -## Decode csv encoded string -Given a sample.yml file of: -```yaml -a: |- - cats,dogs - great,cool as well -``` -then -```bash -yq '.a |= @csvd' sample.yml -``` -will output -```yaml -a: - - cats: great - dogs: cool as well -``` - -## Decode tsv encoded string -Given a sample.yml file of: -```yaml -a: |- - cats dogs - great cool as well -``` -then -```bash -yq '.a |= @tsvd' sample.yml -``` -will output -```yaml -a: - - cats: great - dogs: cool as well -``` - -## Encode value as yaml string -Indent defaults to 2 - -Given a sample.yml file of: -```yaml -a: - cool: - bob: dylan -``` -then -```bash -yq '.b = (.a | to_yaml)' sample.yml -``` -will output -```yaml -a: - cool: - bob: dylan -b: | - cool: - bob: dylan -``` - -## Encode value as yaml string, with custom indentation -You can specify the indentation level as the first parameter. - -Given a sample.yml file of: -```yaml -a: - cool: - bob: dylan -``` -then -```bash -yq '.b = (.a | to_yaml(8))' sample.yml -``` -will output -```yaml -a: - cool: - bob: dylan -b: | - cool: - bob: dylan -``` - -## Decode a yaml encoded string -Given a sample.yml file of: -```yaml -a: 'foo: bar' -``` -then -```bash -yq '.b = (.a | from_yaml)' sample.yml -``` -will output -```yaml -a: 'foo: bar' -b: - foo: bar -``` - -## Update a multiline encoded yaml string -Given a sample.yml file of: -```yaml -a: | - foo: bar - baz: dog - -``` -then -```bash -yq '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml -``` -will output -```yaml -a: | - foo: cat - baz: dog -``` - -## Update a single line encoded yaml string -Given a sample.yml file of: -```yaml -a: 'foo: bar' -``` -then -```bash -yq '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml -``` -will output -```yaml -a: 'foo: cat' -``` - -## Encode array of scalars as csv string -Scalars are strings, numbers and booleans. - -Given a sample.yml file of: -```yaml -- cat -- thing1,thing2 -- true -- 3.40 -``` -then -```bash -yq '@csv' sample.yml -``` -will output -```yaml -cat,"thing1,thing2",true,3.40 -``` - -## Encode array of arrays as csv string -Given a sample.yml file of: -```yaml -- - cat - - thing1,thing2 - - true - - 3.40 -- - dog - - thing3 - - false - - 12 -``` -then -```bash -yq '@csv' sample.yml -``` -will output -```yaml -cat,"thing1,thing2",true,3.40 -dog,thing3,false,12 -``` - -## Encode array of array scalars as tsv string -Scalars are strings, numbers and booleans. - -Given a sample.yml file of: -```yaml -- - cat - - thing1,thing2 - - true - - 3.40 -- - dog - - thing3 - - false - - 12 -``` -then -```bash -yq '@tsv' sample.yml -``` -will output -```yaml -cat thing1,thing2 true 3.40 -dog thing3 false 12 -``` - -## Encode value as xml string -Given a sample.yml file of: -```yaml -a: - cool: - foo: bar - +@id: hi -``` -then -```bash -yq '.a | to_xml' sample.yml -``` -will output -```yaml - - bar - - -``` - -## Encode value as xml string on a single line -Given a sample.yml file of: -```yaml -a: - cool: - foo: bar - +@id: hi -``` -then -```bash -yq '.a | @xml' sample.yml -``` -will output -```yaml -bar - -``` - -## Encode value as xml string with custom indentation -Given a sample.yml file of: -```yaml -a: - cool: - foo: bar - +@id: hi -``` -then -```bash -yq '{"cat": .a | to_xml(1)}' sample.yml -``` -will output -```yaml -cat: | - - bar - -``` - -## Decode a xml encoded string -Given a sample.yml file of: -```yaml -a: bar -``` -then -```bash -yq '.b = (.a | from_xml)' sample.yml -``` -will output -```yaml -a: bar -b: - foo: bar -``` - -## Encode a string to base64 -Given a sample.yml file of: -```yaml -coolData: a special string -``` -then -```bash -yq '.coolData | @base64' sample.yml -``` -will output -```yaml -YSBzcGVjaWFsIHN0cmluZw== -``` - -## Encode a yaml document to base64 -Pipe through @yaml first to convert to a string, then use @base64 to encode it. - -Given a sample.yml file of: -```yaml -a: apple -``` -then -```bash -yq '@yaml | @base64' sample.yml -``` -will output -```yaml -YTogYXBwbGUK -``` - -## Decode a base64 encoded string -Decoded data is assumed to be a string. - -Given a sample.yml file of: -```yaml -coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig== -``` -then -```bash -yq '.coolData | @base64d' sample.yml -``` -will output -```yaml -Works with UTF-16 😊 -``` - -## Decode a base64 encoded yaml document -Pipe through `from_yaml` to parse the decoded base64 string as a yaml document. - -Given a sample.yml file of: -```yaml -coolData: YTogYXBwbGUK -``` -then -```bash -yq '.coolData |= (@base64d | from_yaml)' sample.yml -``` -will output -```yaml -coolData: - a: apple -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/entries.md b/external/yq/pkg/yqlib/doc/operators/entries.md deleted file mode 100755 index a466e43..0000000 --- a/external/yq/pkg/yqlib/doc/operators/entries.md +++ /dev/null @@ -1,141 +0,0 @@ -# Entries - -Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. - -## to_entries Map -Given a sample.yml file of: -```yaml -a: 1 -b: 2 -``` -then -```bash -yq 'to_entries' sample.yml -``` -will output -```yaml -- key: a - value: 1 -- key: b - value: 2 -``` - -## to_entries Array -Given a sample.yml file of: -```yaml -- a -- b -``` -then -```bash -yq 'to_entries' sample.yml -``` -will output -```yaml -- key: 0 - value: a -- key: 1 - value: b -``` - -## to_entries null -Given a sample.yml file of: -```yaml -null -``` -then -```bash -yq 'to_entries' sample.yml -``` -will output -```yaml -``` - -## from_entries map -Given a sample.yml file of: -```yaml -a: 1 -b: 2 -``` -then -```bash -yq 'to_entries | from_entries' sample.yml -``` -will output -```yaml -a: 1 -b: 2 -``` - -## from_entries with numeric key indexes -from_entries always creates a map, even for numeric keys - -Given a sample.yml file of: -```yaml -- a -- b -``` -then -```bash -yq 'to_entries | from_entries' sample.yml -``` -will output -```yaml -0: a -1: b -``` - -## Use with_entries to update keys -Given a sample.yml file of: -```yaml -a: 1 -b: 2 -``` -then -```bash -yq 'with_entries(.key |= "KEY_" + .)' sample.yml -``` -will output -```yaml -KEY_a: 1 -KEY_b: 2 -``` - -## Custom sort map keys -Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back. - -Given a sample.yml file of: -```yaml -a: 1 -c: 3 -b: 2 -``` -then -```bash -yq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml -``` -will output -```yaml -c: 3 -b: 2 -a: 1 -``` - -## Use with_entries to filter the map -Given a sample.yml file of: -```yaml -a: - b: bird -c: - d: dog -``` -then -```bash -yq 'with_entries(select(.value | has("b")))' sample.yml -``` -will output -```yaml -a: - b: bird -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/env-variable-operators.md b/external/yq/pkg/yqlib/doc/operators/env-variable-operators.md deleted file mode 100755 index c616f66..0000000 --- a/external/yq/pkg/yqlib/doc/operators/env-variable-operators.md +++ /dev/null @@ -1,256 +0,0 @@ -# Env Variable Operators - -These operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. - -There are three operators: - -- `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) -- `strenv` which also takes a single environment variable name, and always parses the variable as a string. -- `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). - - -## EnvSubst Options -You can optionally pass envsubst any of the following options: - - - nu: NoUnset, this will fail if there are any referenced variables that are not set - - ne: NoEmpty, this will fail if there are any referenced variables that are empty - - ff: FailFast, this will abort on the first failure (rather than collect all the errors) - -E.g: -`envsubst(ne, ff)` will fail on the first empty variable. - -See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. - -## Tip -To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator -as follows: - -```bash -yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml -``` - - -## Read string environment variable -Running -```bash -myenv="cat meow" yq --null-input '.a = env(myenv)' -``` -will output -```yaml -a: cat meow -``` - -## Read boolean environment variable -Running -```bash -myenv="true" yq --null-input '.a = env(myenv)' -``` -will output -```yaml -a: true -``` - -## Read numeric environment variable -Running -```bash -myenv="12" yq --null-input '.a = env(myenv)' -``` -will output -```yaml -a: 12 -``` - -## Read yaml environment variable -Running -```bash -myenv="{b: fish}" yq --null-input '.a = env(myenv)' -``` -will output -```yaml -a: {b: fish} -``` - -## Read boolean environment variable as a string -Running -```bash -myenv="true" yq --null-input '.a = strenv(myenv)' -``` -will output -```yaml -a: "true" -``` - -## Read numeric environment variable as a string -Running -```bash -myenv="12" yq --null-input '.a = strenv(myenv)' -``` -will output -```yaml -a: "12" -``` - -## Dynamically update a path from an environment variable -The env variable can be any valid yq expression. - -Given a sample.yml file of: -```yaml -a: - b: - - name: dog - - name: cat -``` -then -```bash -pathEnv=".a.b[0].name" valueEnv="moo" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml -``` -will output -```yaml -a: - b: - - name: moo - - name: cat -``` - -## Dynamic key lookup with environment variable -Given a sample.yml file of: -```yaml -cat: meow -dog: woof -``` -then -```bash -myenv="cat" yq '.[env(myenv)]' sample.yml -``` -will output -```yaml -meow -``` - -## Replace strings with envsubst -Running -```bash -myenv="cat" yq --null-input '"the ${myenv} meows" | envsubst' -``` -will output -```yaml -the cat meows -``` - -## Replace strings with envsubst, missing variables -Running -```bash -yq --null-input '"the ${myenvnonexisting} meows" | envsubst' -``` -will output -```yaml -the meows -``` - -## Replace strings with envsubst(nu), missing variables -(nu) not unset, will fail if there are unset (missing) variables - -Running -```bash -yq --null-input '"the ${myenvnonexisting} meows" | envsubst(nu)' -``` -will output -```bash -Error: variable ${myenvnonexisting} not set -``` - -## Replace strings with envsubst(ne), missing variables -(ne) not empty, only validates set variables - -Running -```bash -yq --null-input '"the ${myenvnonexisting} meows" | envsubst(ne)' -``` -will output -```yaml -the meows -``` - -## Replace strings with envsubst(ne), empty variable -(ne) not empty, will fail if a references variable is empty - -Running -```bash -myenv="" yq --null-input '"the ${myenv} meows" | envsubst(ne)' -``` -will output -```bash -Error: variable ${myenv} set but empty -``` - -## Replace strings with envsubst, missing variables with defaults -Running -```bash -yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst' -``` -will output -```yaml -the dog meows -``` - -## Replace strings with envsubst(nu), missing variables with defaults -Having a default specified skips over the missing variable. - -Running -```bash -yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst(nu)' -``` -will output -```yaml -the dog meows -``` - -## Replace strings with envsubst(ne), missing variables with defaults -Fails, because the variable is explicitly set to blank. - -Running -```bash -myEmptyEnv="" yq --null-input '"the ${myEmptyEnv-dog} meows" | envsubst(ne)' -``` -will output -```bash -Error: variable ${myEmptyEnv} set but empty -``` - -## Replace string environment variable in document -Given a sample.yml file of: -```yaml -v: ${myenv} -``` -then -```bash -myenv="cat meow" yq '.v |= envsubst' sample.yml -``` -will output -```yaml -v: cat meow -``` - -## (Default) Return all envsubst errors -By default, all errors are returned at once. - -Running -```bash -yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu)' -``` -will output -```bash -Error: variable ${notThere} not set -variable ${alsoNotThere} not set -``` - -## Fail fast, return the first envsubst error (and abort) -Running -```bash -yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)' -``` -will output -```bash -Error: variable ${notThere} not set -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/equals.md b/external/yq/pkg/yqlib/doc/operators/equals.md deleted file mode 100755 index ca5099b..0000000 --- a/external/yq/pkg/yqlib/doc/operators/equals.md +++ /dev/null @@ -1,133 +0,0 @@ -# Equals / Not Equals - -This is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise. - -``` -.a == .b -``` - -It is most often used with the select operator to find particular nodes: - -``` -select(.a == .b) -``` - -The not equals `!=` operator returns `false` if the LHS is equal to the RHS. - -## Related Operators - -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) - - -## Match string -Given a sample.yml file of: -```yaml -- cat -- goat -- dog -``` -then -```bash -yq '.[] | (. == "*at")' sample.yml -``` -will output -```yaml -true -true -false -``` - -## Don't match string -Given a sample.yml file of: -```yaml -- cat -- goat -- dog -``` -then -```bash -yq '.[] | (. != "*at")' sample.yml -``` -will output -```yaml -false -false -true -``` - -## Match number -Given a sample.yml file of: -```yaml -- 3 -- 4 -- 5 -``` -then -```bash -yq '.[] | (. == 4)' sample.yml -``` -will output -```yaml -false -true -false -``` - -## Don't match number -Given a sample.yml file of: -```yaml -- 3 -- 4 -- 5 -``` -then -```bash -yq '.[] | (. != 4)' sample.yml -``` -will output -```yaml -true -false -true -``` - -## Match nulls -Running -```bash -yq --null-input 'null == ~' -``` -will output -```yaml -true -``` - -## Non existent key doesn't equal a value -Given a sample.yml file of: -```yaml -a: frog -``` -then -```bash -yq 'select(.b != "thing")' sample.yml -``` -will output -```yaml -a: frog -``` - -## Two non existent keys are equal -Given a sample.yml file of: -```yaml -a: frog -``` -then -```bash -yq 'select(.b == .c)' sample.yml -``` -will output -```yaml -a: frog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/error.md b/external/yq/pkg/yqlib/doc/operators/error.md deleted file mode 100755 index 0b7a247..0000000 --- a/external/yq/pkg/yqlib/doc/operators/error.md +++ /dev/null @@ -1,50 +0,0 @@ -# Error - -Use this operation to short-circuit expressions. Useful for validation. - -## Validate a particular value -Given a sample.yml file of: -```yaml -a: hello -``` -then -```bash -yq 'select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")' sample.yml -``` -will output -```bash -Error: .a [hello] is not howdy! -``` - -## Validate the environment variable is a number - invalid -Running -```bash -numberOfCats="please" yq --null-input 'env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")' -``` -will output -```bash -Error: numberOfCats is not a number :( -``` - -## Validate the environment variable is a number - valid -`with` can be a convenient way of encapsulating validation. - -Given a sample.yml file of: -```yaml -name: Bob -favouriteAnimal: cat -``` -then -```bash -numberOfCats="3" yq ' - with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | - .numPets = env(numberOfCats) -' sample.yml -``` -will output -```yaml -name: Bob -favouriteAnimal: cat -numPets: 3 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/eval.md b/external/yq/pkg/yqlib/doc/operators/eval.md deleted file mode 100755 index 1a8a010..0000000 --- a/external/yq/pkg/yqlib/doc/operators/eval.md +++ /dev/null @@ -1,48 +0,0 @@ -# Eval - -Use `eval` to dynamically process an expression - for instance from an environment variable. - -`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. - -Tip: This can be useful way parameterise complex scripts. - -## Dynamically evaluate a path -Given a sample.yml file of: -```yaml -pathExp: .a.b[] | select(.name == "cat") -a: - b: - - name: dog - - name: cat -``` -then -```bash -yq 'eval(.pathExp)' sample.yml -``` -will output -```yaml -name: cat -``` - -## Dynamically update a path from an environment variable -The env variable can be any valid yq expression. - -Given a sample.yml file of: -```yaml -a: - b: - - name: dog - - name: cat -``` -then -```bash -pathEnv=".a.b[0].name" valueEnv="moo" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml -``` -will output -```yaml -a: - b: - - name: moo - - name: cat -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/file-operators.md b/external/yq/pkg/yqlib/doc/operators/file-operators.md deleted file mode 100755 index b1dacd7..0000000 --- a/external/yq/pkg/yqlib/doc/operators/file-operators.md +++ /dev/null @@ -1,74 +0,0 @@ -# File Operators - -File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document). - -Note that the `fileIndex` operator has a short alias of `fi`. - -## Merging files -Note the use of eval-all to ensure all documents are loaded into memory. -```bash -yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml -``` - -## Get filename -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq 'filename' sample.yml -``` -will output -```yaml -sample.yml -``` - -## Get file index -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq 'file_index' sample.yml -``` -will output -```yaml -0 -``` - -## Get file indices of multiple documents -Given a sample.yml file of: -```yaml -a: cat -``` -And another sample another.yml file of: -```yaml -a: cat -``` -then -```bash -yq eval-all 'file_index' sample.yml another.yml -``` -will output -```yaml -0 ---- -1 -``` - -## Get file index alias -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq 'fi' sample.yml -``` -will output -```yaml -0 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/flatten.md b/external/yq/pkg/yqlib/doc/operators/flatten.md deleted file mode 100755 index cb3bc26..0000000 --- a/external/yq/pkg/yqlib/doc/operators/flatten.md +++ /dev/null @@ -1,71 +0,0 @@ -# Flatten -This recursively flattens arrays. - -## Flatten -Recursively flattens all arrays - -Given a sample.yml file of: -```yaml -- 1 -- - 2 -- - - 3 -``` -then -```bash -yq 'flatten' sample.yml -``` -will output -```yaml -- 1 -- 2 -- 3 -``` - -## Flatten with depth of one -Given a sample.yml file of: -```yaml -- 1 -- - 2 -- - - 3 -``` -then -```bash -yq 'flatten(1)' sample.yml -``` -will output -```yaml -- 1 -- 2 -- - 3 -``` - -## Flatten empty array -Given a sample.yml file of: -```yaml -- [] -``` -then -```bash -yq 'flatten' sample.yml -``` -will output -```yaml -[] -``` - -## Flatten array of objects -Given a sample.yml file of: -```yaml -- foo: bar -- - foo: baz -``` -then -```bash -yq 'flatten' sample.yml -``` -will output -```yaml -- foo: bar -- foo: baz -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/group-by.md b/external/yq/pkg/yqlib/doc/operators/group-by.md deleted file mode 100755 index f5169b7..0000000 --- a/external/yq/pkg/yqlib/doc/operators/group-by.md +++ /dev/null @@ -1,56 +0,0 @@ -# Group By - -This is used to group items in an array by an expression. - -## Group by field -Given a sample.yml file of: -```yaml -- foo: 1 - bar: 10 -- foo: 3 - bar: 100 -- foo: 1 - bar: 1 -``` -then -```bash -yq 'group_by(.foo)' sample.yml -``` -will output -```yaml -- - foo: 1 - bar: 10 - - foo: 1 - bar: 1 -- - foo: 3 - bar: 100 -``` - -## Group by field, with nuls -Given a sample.yml file of: -```yaml -- cat: dog -- foo: 1 - bar: 10 -- foo: 3 - bar: 100 -- no: foo for you -- foo: 1 - bar: 1 -``` -then -```bash -yq 'group_by(.foo)' sample.yml -``` -will output -```yaml -- - cat: dog - - no: foo for you -- - foo: 1 - bar: 10 - - foo: 1 - bar: 1 -- - foo: 3 - bar: 100 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/has.md b/external/yq/pkg/yqlib/doc/operators/has.md deleted file mode 100755 index 4a5e294..0000000 --- a/external/yq/pkg/yqlib/doc/operators/has.md +++ /dev/null @@ -1,70 +0,0 @@ -# Has - -This is operation that returns true if the key exists in a map (or index in an array), false otherwise. - -## Has map key -Given a sample.yml file of: -```yaml -- a: yes -- a: ~ -- a: -- b: nope -``` -then -```bash -yq '.[] | has("a")' sample.yml -``` -will output -```yaml -true -true -true -false -``` - -## Select, checking for existence of deep paths -Simply pipe in parent expressions into `has` - -Given a sample.yml file of: -```yaml -- a: - b: - c: cat -- a: - b: - d: dog -``` -then -```bash -yq '.[] | select(.a.b | has("c"))' sample.yml -``` -will output -```yaml -a: - b: - c: cat -``` - -## Has array index -Given a sample.yml file of: -```yaml -- [] -- [1] -- [1, 2] -- [1, null] -- [1, 2, 3] - -``` -then -```bash -yq '.[] | has(1)' sample.yml -``` -will output -```yaml -false -false -true -true -true -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/Main.md b/external/yq/pkg/yqlib/doc/operators/headers/Main.md deleted file mode 100755 index 59284fd..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/Main.md +++ /dev/null @@ -1,63 +0,0 @@ -# NAME - *yq* is a portable command-line YAML processor - -# SYNOPSIS - -yq [eval/eval-all] [expression] files.. - -eval/e - (default) Apply the expression to each document in each yaml file in sequence - -eval-all/ea - Loads all yaml documents of all yaml files and runs expression once - -# DESCRIPTION - -a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. - -This documentation is also available at https://mikefarah.gitbook.io/yq/ -# QUICK GUIDE - -## Read a value: -```bash -yq '.a.b[0].c' file.yaml -``` - -## Pipe from STDIN: -```bash -cat file.yaml | yq '.a.b[0].c' -``` - -## Update a yaml file, inplace -```bash -yq -i '.a.b[0].c = "cool"' file.yaml -``` - -## Update using environment variables -```bash -NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml -``` - -## Merge multiple files -``` -yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml -``` -Note the use of `ea` to evaluate all files at once (instead of in sequence.) - -## Multiple updates to a yaml file -```bash -yq -i ' - .a.b[0].c = "cool" | - .x.y.z = "foobar" | - .person.name = strenv(NAME) -' file.yaml -``` - -See the [documentation](https://mikefarah.gitbook.io/yq/) for more. - -# KNOWN ISSUES / MISSING FEATURES -- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details) -- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell - -# BUGS / ISSUES / FEATURE REQUESTS - -Please visit the GitHub page https://github.com/mikefarah/yq/. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/add.md b/external/yq/pkg/yqlib/doc/operators/headers/add.md deleted file mode 100755 index 4d8524a..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/add.md +++ /dev/null @@ -1,10 +0,0 @@ -# Add - -Add behaves differently according to the type of the LHS: -* arrays: concatenate -* number scalars: arithmetic addition -* string scalars: concatenate -* maps: shallow merge (use the multiply operator (`*`) to deeply merge) - -Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/alternative-default-value.md b/external/yq/pkg/yqlib/doc/operators/headers/alternative-default-value.md deleted file mode 100755 index de5192b..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/alternative-default-value.md +++ /dev/null @@ -1,3 +0,0 @@ -# Alternative (Default value) - -This operator is used to provide alternative (or default) values when a particular expression is either null or false. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md deleted file mode 100755 index 5c6f29e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md +++ /dev/null @@ -1,6 +0,0 @@ -# Anchor and Alias Operators - -Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names). - -`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/array-to-map.md b/external/yq/pkg/yqlib/doc/operators/headers/array-to-map.md deleted file mode 100755 index 0e431a6..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/array-to-map.md +++ /dev/null @@ -1,9 +0,0 @@ -# Array to Map - -Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over. - -Behind the scenes, this is implemented using reduce: - -``` -(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i) -``` diff --git a/external/yq/pkg/yqlib/doc/operators/headers/assign-update.md b/external/yq/pkg/yqlib/doc/operators/headers/assign-update.md deleted file mode 100755 index fc139f3..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/assign-update.md +++ /dev/null @@ -1,13 +0,0 @@ -# Assign (Update) - -This operator is used to update node values. It can be used in either the: - -### plain form: `=` -Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline. - -### relative form: `|=` -This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment. - - -### Flags -- `c` clobber custom tags diff --git a/external/yq/pkg/yqlib/doc/operators/headers/boolean-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/boolean-operators.md deleted file mode 100755 index 6d457cb..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/boolean-operators.md +++ /dev/null @@ -1,17 +0,0 @@ -# Boolean Operators - -The `or` and `and` operators take two parameters and return a boolean result. - -`not` flips a boolean from true to false, or vice versa. - -`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true. - -`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet - -These are most commonly used with the `select` operator to filter particular nodes. - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) diff --git a/external/yq/pkg/yqlib/doc/operators/headers/collect-into-array.md b/external/yq/pkg/yqlib/doc/operators/headers/collect-into-array.md deleted file mode 100755 index defa7ce..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/collect-into-array.md +++ /dev/null @@ -1,4 +0,0 @@ -# Collect into Array - -This creates an array using the expression between the square brackets. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/column.md b/external/yq/pkg/yqlib/doc/operators/headers/column.md deleted file mode 100755 index 6c99974..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/column.md +++ /dev/null @@ -1,3 +0,0 @@ -# Column - -Returns the column of the matching node. Starts from 1, 0 indicates there was no column data. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/comment-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/comment-operators.md deleted file mode 100755 index d7a19d2..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/comment-operators.md +++ /dev/null @@ -1,11 +0,0 @@ -# Comment Operators - -Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples. - -Like the `=` and `|=` assign operators, the same syntax applies when updating comments: - -### plain form: `=` -This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline - -### relative form: `|=` -Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/compare.md b/external/yq/pkg/yqlib/doc/operators/headers/compare.md deleted file mode 100755 index c63f599..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/compare.md +++ /dev/null @@ -1,15 +0,0 @@ -# Compare Operators - -Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. - -The following types are currently supported: - -- numbers -- strings -- datetimes - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) diff --git a/external/yq/pkg/yqlib/doc/operators/headers/contains.md b/external/yq/pkg/yqlib/doc/operators/headers/contains.md deleted file mode 100755 index fad851e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/contains.md +++ /dev/null @@ -1,11 +0,0 @@ -# Contains - -This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring. - -{% hint style="warning" %} - -_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an array like `contains(["cat"])` will return true for an array `["cats"]`. - -See the "Array has a subset array" example below on how to check for a subset. - -{% endhint %} diff --git a/external/yq/pkg/yqlib/doc/operators/headers/create-collect-into-object.md b/external/yq/pkg/yqlib/doc/operators/headers/create-collect-into-object.md deleted file mode 100755 index 85db0fa..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/create-collect-into-object.md +++ /dev/null @@ -1,3 +0,0 @@ -# Create, Collect into Object - -This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/datetime.md b/external/yq/pkg/yqlib/doc/operators/headers/datetime.md deleted file mode 100755 index 36da717..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/datetime.md +++ /dev/null @@ -1,26 +0,0 @@ -# Date Time - -Various operators for parsing and manipulating dates. - -## Date time formattings -This uses the golangs built in time library for parsing and formatting date times. - -When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. - -To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. - -```bash -yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' -``` - -See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options. - - -## Timezones -This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details. - - -## Durations -Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. - -You can durations to time using the `+` operator. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/delete.md b/external/yq/pkg/yqlib/doc/operators/headers/delete.md deleted file mode 100755 index 1a63587..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/delete.md +++ /dev/null @@ -1,3 +0,0 @@ -# Delete - -Deletes matching entries in maps or arrays. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/document-index.md b/external/yq/pkg/yqlib/doc/operators/headers/document-index.md deleted file mode 100755 index 8b2e655..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/document-index.md +++ /dev/null @@ -1,3 +0,0 @@ -# Document Index - -Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/encode-decode.md b/external/yq/pkg/yqlib/doc/operators/headers/encode-decode.md deleted file mode 100755 index f83b45d..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/encode-decode.md +++ /dev/null @@ -1,26 +0,0 @@ -# Encoder / Decoder - -Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. - -Note that you can optionally pass an indent value to the encode functions (see below). - -These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them. - - -| Format | Decode (from string) | Encode (to string) | -| --- | -- | --| -| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml | -| JSON | from_json/@jsond | to_json(i)/@json | -| Properties | from_props/@propsd | to_props/@props | -| CSV | from_csv/@csvd | to_csv/@csv | -| TSV | from_tsv/@tsvd | to_tsv/@tsv | -| XML | from_xml/@xmld | to_xml(i)/@xml | -| Base64 | @base64d | @base64 | - - -See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. - -XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. - - -Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/entries.md b/external/yq/pkg/yqlib/doc/operators/headers/entries.md deleted file mode 100755 index 38ab12d..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/entries.md +++ /dev/null @@ -1,3 +0,0 @@ -# Entries - -Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/env-variable-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/env-variable-operators.md deleted file mode 100755 index b8d795c..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/env-variable-operators.md +++ /dev/null @@ -1,31 +0,0 @@ -# Env Variable Operators - -These operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. - -There are three operators: - -- `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) -- `strenv` which also takes a single environment variable name, and always parses the variable as a string. -- `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). - - -## EnvSubst Options -You can optionally pass envsubst any of the following options: - - - nu: NoUnset, this will fail if there are any referenced variables that are not set - - ne: NoEmpty, this will fail if there are any referenced variables that are empty - - ff: FailFast, this will abort on the first failure (rather than collect all the errors) - -E.g: -`envsubst(ne, ff)` will fail on the first empty variable. - -See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. - -## Tip -To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator -as follows: - -```bash -yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/equals.md b/external/yq/pkg/yqlib/doc/operators/headers/equals.md deleted file mode 100755 index 588cf17..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/equals.md +++ /dev/null @@ -1,22 +0,0 @@ -# Equals / Not Equals - -This is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise. - -``` -.a == .b -``` - -It is most often used with the select operator to find particular nodes: - -``` -select(.a == .b) -``` - -The not equals `!=` operator returns `false` if the LHS is equal to the RHS. - -## Related Operators - -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) -- select operator [here](https://mikefarah.gitbook.io/yq/operators/select) - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/error.md b/external/yq/pkg/yqlib/doc/operators/headers/error.md deleted file mode 100755 index a9fb1a5..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/error.md +++ /dev/null @@ -1,3 +0,0 @@ -# Error - -Use this operation to short-circuit expressions. Useful for validation. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/eval.md b/external/yq/pkg/yqlib/doc/operators/headers/eval.md deleted file mode 100755 index 45c1a22..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/eval.md +++ /dev/null @@ -1,7 +0,0 @@ -# Eval - -Use `eval` to dynamically process an expression - for instance from an environment variable. - -`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. - -Tip: This can be useful way parameterise complex scripts. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/file-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/file-operators.md deleted file mode 100755 index 7729d94..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/file-operators.md +++ /dev/null @@ -1,11 +0,0 @@ -# File Operators - -File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document). - -Note that the `fileIndex` operator has a short alias of `fi`. - -## Merging files -Note the use of eval-all to ensure all documents are loaded into memory. -```bash -yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml -``` diff --git a/external/yq/pkg/yqlib/doc/operators/headers/flatten.md b/external/yq/pkg/yqlib/doc/operators/headers/flatten.md deleted file mode 100755 index 2ac9896..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/flatten.md +++ /dev/null @@ -1,2 +0,0 @@ -# Flatten -This recursively flattens arrays. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/group-by.md b/external/yq/pkg/yqlib/doc/operators/headers/group-by.md deleted file mode 100755 index a890006..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/group-by.md +++ /dev/null @@ -1,3 +0,0 @@ -# Group By - -This is used to group items in an array by an expression. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/has.md b/external/yq/pkg/yqlib/doc/operators/headers/has.md deleted file mode 100755 index 3be66fe..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/has.md +++ /dev/null @@ -1,3 +0,0 @@ -# Has - -This is operation that returns true if the key exists in a map (or index in an array), false otherwise. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/keys.md b/external/yq/pkg/yqlib/doc/operators/headers/keys.md deleted file mode 100755 index 6362260..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/keys.md +++ /dev/null @@ -1,3 +0,0 @@ -# Keys - -Use the `keys` operator to return map keys or array indices. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/length.md b/external/yq/pkg/yqlib/doc/operators/headers/length.md deleted file mode 100755 index cb5bf09..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/length.md +++ /dev/null @@ -1,3 +0,0 @@ -# Length - -Returns the lengths of the nodes. Length is defined according to the type of the node. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/line.md b/external/yq/pkg/yqlib/doc/operators/headers/line.md deleted file mode 100755 index 4b9bca0..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/line.md +++ /dev/null @@ -1,3 +0,0 @@ -# Line - -Returns the line of the matching node. Starts from 1, 0 indicates there was no line data. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/load.md b/external/yq/pkg/yqlib/doc/operators/headers/load.md deleted file mode 100755 index 9989f2f..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/load.md +++ /dev/null @@ -1,46 +0,0 @@ -# Load - -The load operators allows you to load in content from another file. - -Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. - -You can load files of the following supported types: - -|Format | Load Operator | -| --- | --- | -| Yaml | load | -| XML | load_xml | -| Properties | load_props | -| Plain String | load_str | -| Base64 | load_base64 | - -## Samples files for tests: - -### yaml - -`../../examples/thing.yml`: - -```yaml -a: apple is included -b: cool -``` - -### xml -`small.xml`: - -```xml -is some xml -``` - -### properties -`small.properties`: - -```properties -this.is = a properties file -``` - -### base64 -`base64.txt`: -``` -bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u -``` diff --git a/external/yq/pkg/yqlib/doc/operators/headers/map.md b/external/yq/pkg/yqlib/doc/operators/headers/map.md deleted file mode 100755 index e5fef62..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/map.md +++ /dev/null @@ -1,3 +0,0 @@ -# Map - -Maps values of an array. Use `map_values` to map values of an object. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/multiply-merge.md b/external/yq/pkg/yqlib/doc/operators/headers/multiply-merge.md deleted file mode 100755 index 1dad37e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/multiply-merge.md +++ /dev/null @@ -1,37 +0,0 @@ -# Multiply (Merge) - -Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. - -## Objects and arrays - merging -Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged. - -Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. - -### Merge Flags -You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below - -- `+` append arrays -- `d` deeply merge arrays -- `?` only merge _existing_ fields -- `n` only merge _new_ fields -- `c` clobber custom tags - - -### Merge two files together -This uses the load operator to merge file2 into file1. -```bash -yq '. *= load("file2.yml")' file1.yml -``` - -### Merging all files -Note the use of `eval-all` to ensure all documents are loaded into memory. - -```bash -yq eval-all '. as $item ireduce ({}; . * $item )' *.yml -``` - -# Merging complex arrays together by a key field -By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array. - -For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key) - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/parent.md b/external/yq/pkg/yqlib/doc/operators/headers/parent.md deleted file mode 100755 index 9e60cd8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/parent.md +++ /dev/null @@ -1,3 +0,0 @@ -# Parent - -Parent simply returns the parent nodes of the matching nodes. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/path.md b/external/yq/pkg/yqlib/doc/operators/headers/path.md deleted file mode 100755 index 252de2c..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/path.md +++ /dev/null @@ -1,8 +0,0 @@ -# Path - -The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node. - -You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key. - -Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/pick.md b/external/yq/pkg/yqlib/doc/operators/headers/pick.md deleted file mode 100755 index 153675e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/pick.md +++ /dev/null @@ -1,5 +0,0 @@ -# Pick - -Filter a map by the specified list of keys. Map is returned with the key in the order of the pick list. - -Similarly, filter an array by the specified list of indices. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/pipe.md b/external/yq/pkg/yqlib/doc/operators/headers/pipe.md deleted file mode 100755 index cce80d7..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/pipe.md +++ /dev/null @@ -1,3 +0,0 @@ -# Pipe - -Pipe the results of an expression into another. Like the bash operator. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/recursive-descent-glob.md b/external/yq/pkg/yqlib/doc/operators/headers/recursive-descent-glob.md deleted file mode 100755 index 90e0d8a..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/recursive-descent-glob.md +++ /dev/null @@ -1,21 +0,0 @@ -# Recursive Descent (Glob) - -This operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the - -## match values form `..` -This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values. - -For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys: - -```bash -yq '.. style= "flow"' file.yaml -``` - -## match values and map keys form `...` -The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases. - -For instance to set the `style` of all nodes in a yaml doc, including the map keys: - -```bash -yq '... style= "flow"' file.yaml -``` \ No newline at end of file diff --git a/external/yq/pkg/yqlib/doc/operators/headers/reduce.md b/external/yq/pkg/yqlib/doc/operators/headers/reduce.md deleted file mode 100755 index 4020ae8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/reduce.md +++ /dev/null @@ -1,22 +0,0 @@ -# Reduce - -Reduce is a powerful way to process a collection of data into a new form. - -``` - as $ ireduce (; ) -``` - -e.g. - -``` -.[] as $item ireduce (0; . + $item) -``` - -On the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements. - -On the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. - -## yq vs jq syntax -Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`). - -To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/reverse.md b/external/yq/pkg/yqlib/doc/operators/headers/reverse.md deleted file mode 100755 index 50602b3..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/reverse.md +++ /dev/null @@ -1,3 +0,0 @@ -# Reverse - -Reverses the order of the items in an array diff --git a/external/yq/pkg/yqlib/doc/operators/headers/select.md b/external/yq/pkg/yqlib/doc/operators/headers/select.md deleted file mode 100755 index d2205a8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/select.md +++ /dev/null @@ -1,9 +0,0 @@ -# Select - -Select is used to filter arrays and maps by a boolean expression. - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) diff --git a/external/yq/pkg/yqlib/doc/operators/headers/slice-array.md b/external/yq/pkg/yqlib/doc/operators/headers/slice-array.md deleted file mode 100755 index 7f2487c..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/slice-array.md +++ /dev/null @@ -1,5 +0,0 @@ -# Slice/Splice Array - -The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. - -You may leave out the first or second number, which will will refer to the start or end of the array respectively. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/sort-keys.md b/external/yq/pkg/yqlib/doc/operators/headers/sort-keys.md deleted file mode 100755 index 31b0b6d..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/sort-keys.md +++ /dev/null @@ -1,16 +0,0 @@ -# Sort Keys - -The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). - -Sort is particularly useful for diffing two different yaml documents: - -```bash -yq -i -P 'sort_keys(..)' file1.yml -yq -i -P 'sort_keys(..)' file2.yml -diff file1.yml file2.yml -``` - -Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors. - -For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`. -See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/sort.md b/external/yq/pkg/yqlib/doc/operators/headers/sort.md deleted file mode 100755 index 1022b80..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/sort.md +++ /dev/null @@ -1,8 +0,0 @@ -# Sort - -Sorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield). - -To sort by descending order, pipe the results through the `reverse` operator after sorting. - -Note that at this stage, `yq` only sorts scalar fields. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/split-into-documents.md b/external/yq/pkg/yqlib/doc/operators/headers/split-into-documents.md deleted file mode 100755 index e62712d..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/split-into-documents.md +++ /dev/null @@ -1,3 +0,0 @@ -# Split into Documents - -This operator splits all matches into separate documents diff --git a/external/yq/pkg/yqlib/doc/operators/headers/string-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/string-operators.md deleted file mode 100755 index e6e2f5e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/string-operators.md +++ /dev/null @@ -1,57 +0,0 @@ -# String Operators - -## RegEx -This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax. - -Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`. - -### match(regEx) -This operator returns the substring match details of the given regEx. - -### capture(regEx) -Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing. - -## test(regEx) -Returns true if the string matches the RegEx, false otherwise. - -## sub(regEx, replacement) -Substitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second is a what to replace those matches with. This can refer to capture groups from the first RegEx. - -## String blocks, bash and newlines -Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_. - -For instance to get this yaml: - -``` -a: | - cat -``` - -Using `$( exp )` wont work, as it will trim the trailing new line. - -``` -m=$(echo "cat\n") yq -n '.a = strenv(m)' -a: cat -``` - -However, using printf works: -``` -printf -v m "cat\n" ; m="$m" yq -n '.a = strenv(m)' -a: | - cat -``` - -As well as having multiline expressions: -``` -m="cat -" yq -n '.a = strenv(m)' -a: | - cat -``` - -Similarly, if you're trying to set the content from a file, and want a trailing new line: - -``` -IFS= read -rd '' output < <(cat my_file) -output=$output ./yq '.data.values = strenv(output)' first.yml -``` diff --git a/external/yq/pkg/yqlib/doc/operators/headers/style.md b/external/yq/pkg/yqlib/doc/operators/headers/style.md deleted file mode 100755 index c31c328..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/style.md +++ /dev/null @@ -1,3 +0,0 @@ -# Style - -The style operator can be used to get or set the style of nodes (e.g. string style, yaml style) diff --git a/external/yq/pkg/yqlib/doc/operators/headers/subtract.md b/external/yq/pkg/yqlib/doc/operators/headers/subtract.md deleted file mode 100755 index 37b12b8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/subtract.md +++ /dev/null @@ -1,3 +0,0 @@ -# Subtract - -You can use subtract to subtract numbers, as well as removing elements from an array. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/tag.md b/external/yq/pkg/yqlib/doc/operators/headers/tag.md deleted file mode 100755 index 99d0347..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/tag.md +++ /dev/null @@ -1,3 +0,0 @@ -# Tag - -The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`). diff --git a/external/yq/pkg/yqlib/doc/operators/headers/traverse-read.md b/external/yq/pkg/yqlib/doc/operators/headers/traverse-read.md deleted file mode 100755 index 6c01240..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/traverse-read.md +++ /dev/null @@ -1,3 +0,0 @@ -# Traverse (Read) - -This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/union.md b/external/yq/pkg/yqlib/doc/operators/headers/union.md deleted file mode 100755 index 7967b9c..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/union.md +++ /dev/null @@ -1,3 +0,0 @@ -# Union - -This operator is used to combine different results together. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/unique.md b/external/yq/pkg/yqlib/doc/operators/headers/unique.md deleted file mode 100755 index 7a11ce4..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/unique.md +++ /dev/null @@ -1,4 +0,0 @@ -# Unique - -This is used to filter out duplicated items in an array. Note that the original order of the array is maintained. - diff --git a/external/yq/pkg/yqlib/doc/operators/headers/variable-operators.md b/external/yq/pkg/yqlib/doc/operators/headers/variable-operators.md deleted file mode 100755 index fd91848..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/variable-operators.md +++ /dev/null @@ -1,5 +0,0 @@ -# Variable Operators - -Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). - -Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. diff --git a/external/yq/pkg/yqlib/doc/operators/headers/with.md b/external/yq/pkg/yqlib/doc/operators/headers/with.md deleted file mode 100755 index f83adbd..0000000 --- a/external/yq/pkg/yqlib/doc/operators/headers/with.md +++ /dev/null @@ -1,3 +0,0 @@ -# With - -Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context. diff --git a/external/yq/pkg/yqlib/doc/operators/keys.md b/external/yq/pkg/yqlib/doc/operators/keys.md deleted file mode 100755 index 4dec429..0000000 --- a/external/yq/pkg/yqlib/doc/operators/keys.md +++ /dev/null @@ -1,154 +0,0 @@ -# Keys - -Use the `keys` operator to return map keys or array indices. - -## Map keys -Given a sample.yml file of: -```yaml -dog: woof -cat: meow -``` -then -```bash -yq 'keys' sample.yml -``` -will output -```yaml -- dog -- cat -``` - -## Array keys -Given a sample.yml file of: -```yaml -- apple -- banana -``` -then -```bash -yq 'keys' sample.yml -``` -will output -```yaml -- 0 -- 1 -``` - -## Retrieve array key -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq '.[1] | key' sample.yml -``` -will output -```yaml -1 -``` - -## Retrieve map key -Given a sample.yml file of: -```yaml -a: thing -``` -then -```bash -yq '.a | key' sample.yml -``` -will output -```yaml -a -``` - -## No key -Given a sample.yml file of: -```yaml -{} -``` -then -```bash -yq 'key' sample.yml -``` -will output -```yaml -``` - -## Update map key -Given a sample.yml file of: -```yaml -a: - x: 3 - y: 4 -``` -then -```bash -yq '(.a.x | key) = "meow"' sample.yml -``` -will output -```yaml -a: - meow: 3 - y: 4 -``` - -## Get comment from map key -Given a sample.yml file of: -```yaml -a: - # comment on key - x: 3 - y: 4 -``` -then -```bash -yq '.a.x | key | headComment' sample.yml -``` -will output -```yaml -comment on key -``` - -## Check node is a key -Given a sample.yml file of: -```yaml -a: - b: - - cat - c: frog -``` -then -```bash -yq '[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]' sample.yml -``` -will output -```yaml -- p: "" - isKey: false - tag: '!!map' -- p: a - isKey: true - tag: '!!str' -- p: a - isKey: false - tag: '!!map' -- p: a.b - isKey: true - tag: '!!str' -- p: a.b - isKey: false - tag: '!!seq' -- p: a.b.0 - isKey: false - tag: '!!str' -- p: a.c - isKey: true - tag: '!!str' -- p: a.c - isKey: false - tag: '!!str' -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/length.md b/external/yq/pkg/yqlib/doc/operators/length.md deleted file mode 100755 index 6315e5b..0000000 --- a/external/yq/pkg/yqlib/doc/operators/length.md +++ /dev/null @@ -1,70 +0,0 @@ -# Length - -Returns the lengths of the nodes. Length is defined according to the type of the node. - -## String length -returns length of string - -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a | length' sample.yml -``` -will output -```yaml -3 -``` - -## null length -Given a sample.yml file of: -```yaml -a: null -``` -then -```bash -yq '.a | length' sample.yml -``` -will output -```yaml -0 -``` - -## Map length -returns number of entries - -Given a sample.yml file of: -```yaml -a: cat -c: dog -``` -then -```bash -yq 'length' sample.yml -``` -will output -```yaml -2 -``` - -## Array length -returns number of elements - -Given a sample.yml file of: -```yaml -- 2 -- 4 -- 6 -- 8 -``` -then -```bash -yq 'length' sample.yml -``` -will output -```yaml -4 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/line.md b/external/yq/pkg/yqlib/doc/operators/line.md deleted file mode 100755 index 9fc4cd0..0000000 --- a/external/yq/pkg/yqlib/doc/operators/line.md +++ /dev/null @@ -1,62 +0,0 @@ -# Line - -Returns the line of the matching node. Starts from 1, 0 indicates there was no line data. - -## Returns line of _value_ node -Given a sample.yml file of: -```yaml -a: cat -b: - c: cat -``` -then -```bash -yq '.b | line' sample.yml -``` -will output -```yaml -3 -``` - -## Returns line of _key_ node -Pipe through the key operator to get the line of the key - -Given a sample.yml file of: -```yaml -a: cat -b: - c: cat -``` -then -```bash -yq '.b | key| line' sample.yml -``` -will output -```yaml -2 -``` - -## First line is 1 -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a | line' sample.yml -``` -will output -```yaml -1 -``` - -## No line data is 0 -Running -```bash -yq --null-input '{"a": "new entry"} | line' -``` -will output -```yaml -0 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/load.md b/external/yq/pkg/yqlib/doc/operators/load.md deleted file mode 100755 index 79dc014..0000000 --- a/external/yq/pkg/yqlib/doc/operators/load.md +++ /dev/null @@ -1,194 +0,0 @@ -# Load - -The load operators allows you to load in content from another file. - -Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. - -You can load files of the following supported types: - -|Format | Load Operator | -| --- | --- | -| Yaml | load | -| XML | load_xml | -| Properties | load_props | -| Plain String | load_str | -| Base64 | load_base64 | - -## Samples files for tests: - -### yaml - -`../../examples/thing.yml`: - -```yaml -a: apple is included -b: cool -``` - -### xml -`small.xml`: - -```xml -is some xml -``` - -### properties -`small.properties`: - -```properties -this.is = a properties file -``` - -### base64 -`base64.txt`: -``` -bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u -``` - -## Simple example -Given a sample.yml file of: -```yaml -myFile: ../../examples/thing.yml -``` -then -```bash -yq 'load(.myFile)' sample.yml -``` -will output -```yaml -a: apple is included -b: cool. -``` - -## Replace node with referenced file -Note that you can modify the filename in the load operator if needed. - -Given a sample.yml file of: -```yaml -something: - file: thing.yml -``` -then -```bash -yq '.something |= load("../../examples/" + .file)' sample.yml -``` -will output -```yaml -something: - a: apple is included - b: cool. -``` - -## Replace _all_ nodes with referenced file -Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. - -Given a sample.yml file of: -```yaml -something: - file: thing.yml -over: - here: - - file: thing.yml -``` -then -```bash -yq '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml -``` -will output -```yaml -something: - a: apple is included - b: cool. -over: - here: - - a: apple is included - b: cool. -``` - -## Replace node with referenced file as string -This will work for any text based file - -Given a sample.yml file of: -```yaml -something: - file: thing.yml -``` -then -```bash -yq '.something |= load_str("../../examples/" + .file)' sample.yml -``` -will output -```yaml -something: |- - a: apple is included - b: cool. -``` - -## Load from XML -Given a sample.yml file of: -```yaml -cool: things -``` -then -```bash -yq '.more_stuff = load_xml("../../examples/small.xml")' sample.yml -``` -will output -```yaml -cool: things -more_stuff: - this: is some xml -``` - -## Load from Properties -Given a sample.yml file of: -```yaml -cool: things -``` -then -```bash -yq '.more_stuff = load_props("../../examples/small.properties")' sample.yml -``` -will output -```yaml -cool: things -more_stuff: - this: - is: a properties file -``` - -## Merge from properties -This can be used as a convenient way to update a yaml document - -Given a sample.yml file of: -```yaml -this: - is: from yaml - cool: ay -``` -then -```bash -yq '. *= load_props("../../examples/small.properties")' sample.yml -``` -will output -```yaml -this: - is: a properties file - cool: ay -``` - -## Load from base64 encoded file -Given a sample.yml file of: -```yaml -cool: things -``` -then -```bash -yq '.more_stuff = load_base64("../../examples/base64.txt")' sample.yml -``` -will output -```yaml -cool: things -more_stuff: my secret chilli recipe is.... -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/map.md b/external/yq/pkg/yqlib/doc/operators/map.md deleted file mode 100755 index 991167e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/map.md +++ /dev/null @@ -1,40 +0,0 @@ -# Map - -Maps values of an array. Use `map_values` to map values of an object. - -## Map array -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq 'map(. + 1)' sample.yml -``` -will output -```yaml -- 2 -- 3 -- 4 -``` - -## Map object values -Given a sample.yml file of: -```yaml -a: 1 -b: 2 -c: 3 -``` -then -```bash -yq 'map_values(. + 1)' sample.yml -``` -will output -```yaml -a: 2 -b: 3 -c: 4 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/multiply-merge.md b/external/yq/pkg/yqlib/doc/operators/multiply-merge.md deleted file mode 100755 index 6db1266..0000000 --- a/external/yq/pkg/yqlib/doc/operators/multiply-merge.md +++ /dev/null @@ -1,501 +0,0 @@ -# Multiply (Merge) - -Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. - -## Objects and arrays - merging -Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged. - -Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. - -### Merge Flags -You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below - -- `+` append arrays -- `d` deeply merge arrays -- `?` only merge _existing_ fields -- `n` only merge _new_ fields -- `c` clobber custom tags - - -### Merge two files together -This uses the load operator to merge file2 into file1. -```bash -yq '. *= load("file2.yml")' file1.yml -``` - -### Merging all files -Note the use of `eval-all` to ensure all documents are loaded into memory. - -```bash -yq eval-all '. as $item ireduce ({}; . * $item )' *.yml -``` - -# Merging complex arrays together by a key field -By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array. - -For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key) - - -## Multiply integers -Given a sample.yml file of: -```yaml -a: 3 -b: 4 -``` -then -```bash -yq '.a *= .b' sample.yml -``` -will output -```yaml -a: 12 -b: 4 -``` - -## Merge objects together, returning merged result only -Given a sample.yml file of: -```yaml -a: - field: me - fieldA: cat -b: - field: - g: wizz - fieldB: dog -``` -then -```bash -yq '.a * .b' sample.yml -``` -will output -```yaml -field: - g: wizz -fieldA: cat -fieldB: dog -``` - -## Merge objects together, returning parent object -Given a sample.yml file of: -```yaml -a: - field: me - fieldA: cat -b: - field: - g: wizz - fieldB: dog -``` -then -```bash -yq '. * {"a":.b}' sample.yml -``` -will output -```yaml -a: - field: - g: wizz - fieldA: cat - fieldB: dog -b: - field: - g: wizz - fieldB: dog -``` - -## Merge keeps style of LHS -Given a sample.yml file of: -```yaml -a: {things: great} -b: - also: "me" -``` -then -```bash -yq '. * {"a":.b}' sample.yml -``` -will output -```yaml -a: {things: great, also: "me"} -b: - also: "me" -``` - -## Merge arrays -Given a sample.yml file of: -```yaml -a: - - 1 - - 2 - - 3 -b: - - 3 - - 4 - - 5 -``` -then -```bash -yq '. * {"a":.b}' sample.yml -``` -will output -```yaml -a: - - 3 - - 4 - - 5 -b: - - 3 - - 4 - - 5 -``` - -## Merge, only existing fields -Given a sample.yml file of: -```yaml -a: - thing: one - cat: frog -b: - missing: two - thing: two -``` -then -```bash -yq '.a *? .b' sample.yml -``` -will output -```yaml -thing: two -cat: frog -``` - -## Merge, only new fields -Given a sample.yml file of: -```yaml -a: - thing: one - cat: frog -b: - missing: two - thing: two -``` -then -```bash -yq '.a *n .b' sample.yml -``` -will output -```yaml -thing: one -cat: frog -missing: two -``` - -## Merge, appending arrays -Given a sample.yml file of: -```yaml -a: - array: - - 1 - - 2 - - animal: dog - value: coconut -b: - array: - - 3 - - 4 - - animal: cat - value: banana -``` -then -```bash -yq '.a *+ .b' sample.yml -``` -will output -```yaml -array: - - 1 - - 2 - - animal: dog - - 3 - - 4 - - animal: cat -value: banana -``` - -## Merge, only existing fields, appending arrays -Given a sample.yml file of: -```yaml -a: - thing: - - 1 - - 2 -b: - thing: - - 3 - - 4 - another: - - 1 -``` -then -```bash -yq '.a *?+ .b' sample.yml -``` -will output -```yaml -thing: - - 1 - - 2 - - 3 - - 4 -``` - -## Merge, deeply merging arrays -Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second. - -Given a sample.yml file of: -```yaml -a: - - name: fred - age: 12 - - name: bob - age: 32 -b: - - name: fred - age: 34 -``` -then -```bash -yq '.a *d .b' sample.yml -``` -will output -```yaml -- name: fred - age: 34 -- name: bob - age: 32 -``` - -## Merge arrays of objects together, matching on a key - -This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below. - -It merges in the array provided in the second file into the first - matching on equal keys. - -Explanation: - -The approach, at a high level, is to reduce into a merged map (keyed by the unique key) -and then convert that back into an array. - -First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by. -The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. - -Next, we convert the map back to an array, using reduce again, concatenating all the map values together. - -Finally, we set the result of the merged array back into the first doc. - -Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) - - -Given a sample.yml file of: -```yaml -myArray: - - a: apple - b: appleB - - a: kiwi - b: kiwiB - - a: banana - b: bananaB -something: else -``` -And another sample another.yml file of: -```yaml -newArray: - - a: banana - c: bananaC - - a: apple - b: appleB2 - - a: dingo - c: dingoC -``` -then -```bash -idPath=".a" originalPath=".myArray" otherPath=".newArray" yq eval-all ' -( - (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap - | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) -) as $mergedArray -| select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray -' sample.yml another.yml -``` -will output -```yaml -myArray: - - a: apple - b: appleB2 - - a: kiwi - b: kiwiB - - a: banana - b: bananaB - c: bananaC - - a: dingo - c: dingoC -something: else -``` - -## Merge to prefix an element -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq '. * {"a": {"c": .a}}' sample.yml -``` -will output -```yaml -a: - c: cat -b: dog -``` - -## Merge with simple aliases -Given a sample.yml file of: -```yaml -a: &cat - c: frog -b: - f: *cat -c: - g: thongs -``` -then -```bash -yq '.c * .b' sample.yml -``` -will output -```yaml -g: thongs -f: *cat -``` - -## Merge copies anchor names -Given a sample.yml file of: -```yaml -a: - c: &cat frog -b: - f: *cat -c: - g: thongs -``` -then -```bash -yq '.c * .a' sample.yml -``` -will output -```yaml -g: thongs -c: &cat frog -``` - -## Merge with merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar * .foobarList' sample.yml -``` -will output -```yaml -c: foobarList_c -!!merge <<: - - *foo - - *bar -thing: foobar_thing -b: foobarList_b -``` - -## Custom types: that are really numbers -When custom tags are encountered, yq will try to decode the underlying type. - -Given a sample.yml file of: -```yaml -a: !horse 2 -b: !goat 3 -``` -then -```bash -yq '.a = .a * .b' sample.yml -``` -will output -```yaml -a: !horse 6 -b: !goat 3 -``` - -## Custom types: that are really maps -Custom tags will be maintained. - -Given a sample.yml file of: -```yaml -a: !horse - cat: meow -b: !goat - dog: woof -``` -then -```bash -yq '.a = .a * .b' sample.yml -``` -will output -```yaml -a: !horse - cat: meow - dog: woof -b: !goat - dog: woof -``` - -## Custom types: clobber tags -Use the `c` option to clobber custom tags. Note that the second tag is now used - -Given a sample.yml file of: -```yaml -a: !horse - cat: meow -b: !goat - dog: woof -``` -then -```bash -yq '.a *=c .b' sample.yml -``` -will output -```yaml -a: !goat - cat: meow - dog: woof -b: !goat - dog: woof -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/parent.md b/external/yq/pkg/yqlib/doc/operators/parent.md deleted file mode 100755 index 0ef18d5..0000000 --- a/external/yq/pkg/yqlib/doc/operators/parent.md +++ /dev/null @@ -1,52 +0,0 @@ -# Parent - -Parent simply returns the parent nodes of the matching nodes. - -## Simple example -Given a sample.yml file of: -```yaml -a: - nested: cat -``` -then -```bash -yq '.a.nested | parent' sample.yml -``` -will output -```yaml -nested: cat -``` - -## Parent of nested matches -Given a sample.yml file of: -```yaml -a: - fruit: apple - name: bob -b: - fruit: banana - name: sam -``` -then -```bash -yq '.. | select(. == "banana") | parent' sample.yml -``` -will output -```yaml -fruit: banana -name: sam -``` - -## No parent -Given a sample.yml file of: -```yaml -{} -``` -then -```bash -yq 'parent' sample.yml -``` -will output -```yaml -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/path.md b/external/yq/pkg/yqlib/doc/operators/path.md deleted file mode 100755 index 1e0076a..0000000 --- a/external/yq/pkg/yqlib/doc/operators/path.md +++ /dev/null @@ -1,234 +0,0 @@ -# Path - -The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node. - -You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key. - -Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays. - - -## Map path -Given a sample.yml file of: -```yaml -a: - b: cat -``` -then -```bash -yq '.a.b | path' sample.yml -``` -will output -```yaml -- a -- b -``` - -## Get map key -Given a sample.yml file of: -```yaml -a: - b: cat -``` -then -```bash -yq '.a.b | path | .[-1]' sample.yml -``` -will output -```yaml -b -``` - -## Array path -Given a sample.yml file of: -```yaml -a: - - cat - - dog -``` -then -```bash -yq '.a.[] | select(. == "dog") | path' sample.yml -``` -will output -```yaml -- a -- 1 -``` - -## Get array index -Given a sample.yml file of: -```yaml -a: - - cat - - dog -``` -then -```bash -yq '.a.[] | select(. == "dog") | path | .[-1]' sample.yml -``` -will output -```yaml -1 -``` - -## Print path and value -Given a sample.yml file of: -```yaml -a: - - cat - - dog - - frog -``` -then -```bash -yq '.a[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml -``` -will output -```yaml -- path: - - a - - 1 - value: dog -- path: - - a - - 2 - value: frog -``` - -## Set path -Given a sample.yml file of: -```yaml -a: - b: cat -``` -then -```bash -yq 'setpath(["a", "b"]; "things")' sample.yml -``` -will output -```yaml -a: - b: things -``` - -## Set on empty document -Running -```bash -yq --null-input 'setpath(["a", "b"]; "things")' -``` -will output -```yaml -a: - b: things -``` - -## Set path to prune deep paths -Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object, - -Given a sample.yml file of: -```yaml -parentA: bob -parentB: - child1: i am child1 - child2: i am child2 -parentC: - child1: me child1 - child2: me child2 -``` -then -```bash -yq '(.parentB.child2, .parentC.child1) as $i - ireduce({}; setpath($i | path; $i))' sample.yml -``` -will output -```yaml -parentB: - child2: i am child2 -parentC: - child1: me child1 -``` - -## Set array path -Given a sample.yml file of: -```yaml -a: - - cat - - frog -``` -then -```bash -yq 'setpath(["a", 0]; "things")' sample.yml -``` -will output -```yaml -a: - - things - - frog -``` - -## Set array path empty -Running -```bash -yq --null-input 'setpath(["a", 0]; "things")' -``` -will output -```yaml -a: - - things -``` - -## Delete path -Notice delpaths takes an _array_ of paths. - -Given a sample.yml file of: -```yaml -a: - b: cat - c: dog - d: frog -``` -then -```bash -yq 'delpaths([["a", "c"], ["a", "d"]])' sample.yml -``` -will output -```yaml -a: - b: cat -``` - -## Delete array path -Given a sample.yml file of: -```yaml -a: - - cat - - frog -``` -then -```bash -yq 'delpaths([["a", 0]])' sample.yml -``` -will output -```yaml -a: - - frog -``` - -## Delete - wrong parameter -delpaths does not work with a single path array - -Given a sample.yml file of: -```yaml -a: - - cat - - frog -``` -then -```bash -yq 'delpaths(["a", 0])' sample.yml -``` -will output -```bash -Error: DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [["a", "b"]] -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/pick.md b/external/yq/pkg/yqlib/doc/operators/pick.md deleted file mode 100755 index 45bccc8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/pick.md +++ /dev/null @@ -1,47 +0,0 @@ -# Pick - -Filter a map by the specified list of keys. Map is returned with the key in the order of the pick list. - -Similarly, filter an array by the specified list of indices. - -## Pick keys from map -Note that the order of the keys matches the pick order and non existent keys are skipped. - -Given a sample.yml file of: -```yaml -myMap: - cat: meow - dog: bark - thing: hamster - hamster: squeek -``` -then -```bash -yq '.myMap |= pick(["hamster", "cat", "goat"])' sample.yml -``` -will output -```yaml -myMap: - hamster: squeek - cat: meow -``` - -## Pick indices from array -Note that the order of the indexes matches the pick order and non existent indexes are skipped. - -Given a sample.yml file of: -```yaml -- cat -- leopard -- lion -``` -then -```bash -yq 'pick([2, 0, 734, -5])' sample.yml -``` -will output -```yaml -- lion -- cat -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/pipe.md b/external/yq/pkg/yqlib/doc/operators/pipe.md deleted file mode 100755 index 0821554..0000000 --- a/external/yq/pkg/yqlib/doc/operators/pipe.md +++ /dev/null @@ -1,37 +0,0 @@ -# Pipe - -Pipe the results of an expression into another. Like the bash operator. - -## Simple Pipe -Given a sample.yml file of: -```yaml -a: - b: cat -``` -then -```bash -yq '.a | .b' sample.yml -``` -will output -```yaml -cat -``` - -## Multiple updates -Given a sample.yml file of: -```yaml -a: cow -b: sheep -c: same -``` -then -```bash -yq '.a = "cat" | .b = "dog"' sample.yml -``` -will output -```yaml -a: cat -b: dog -c: same -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/recursive-descent-glob.md b/external/yq/pkg/yqlib/doc/operators/recursive-descent-glob.md deleted file mode 100755 index 1bb1656..0000000 --- a/external/yq/pkg/yqlib/doc/operators/recursive-descent-glob.md +++ /dev/null @@ -1,156 +0,0 @@ -# Recursive Descent (Glob) - -This operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the - -## match values form `..` -This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values. - -For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys: - -```bash -yq '.. style= "flow"' file.yaml -``` - -## match values and map keys form `...` -The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases. - -For instance to set the `style` of all nodes in a yaml doc, including the map keys: - -```bash -yq '... style= "flow"' file.yaml -``` -## Recurse map (values only) -Given a sample.yml file of: -```yaml -a: frog -``` -then -```bash -yq '..' sample.yml -``` -will output -```yaml -a: frog -frog -``` - -## Recursively find nodes with keys -Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression. - -Given a sample.yml file of: -```yaml -a: - name: frog - b: - name: blog - age: 12 -``` -then -```bash -yq '[.. | select(has("name"))]' sample.yml -``` -will output -```yaml -- name: frog - b: - name: blog - age: 12 -- name: blog - age: 12 -``` - -## Recursively find nodes with values -Given a sample.yml file of: -```yaml -a: - nameA: frog - b: - nameB: frog - age: 12 -``` -then -```bash -yq '.. | select(. == "frog")' sample.yml -``` -will output -```yaml -frog -frog -``` - -## Recurse map (values and keys) -Note that the map key appears in the results - -Given a sample.yml file of: -```yaml -a: frog -``` -then -```bash -yq '...' sample.yml -``` -will output -```yaml -a: frog -a -frog -``` - -## Aliases are not traversed -Given a sample.yml file of: -```yaml -a: &cat - c: frog -b: *cat -``` -then -```bash -yq '[..]' sample.yml -``` -will output -```yaml -- a: &cat - c: frog - b: *cat -- &cat - c: frog -- frog -- *cat -``` - -## Merge docs are not traversed -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar | [..]' sample.yml -``` -will output -```yaml -- c: foobar_c - !!merge <<: *foo - thing: foobar_thing -- foobar_c -- *foo -- foobar_thing -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/reduce.md b/external/yq/pkg/yqlib/doc/operators/reduce.md deleted file mode 100755 index 80affbd..0000000 --- a/external/yq/pkg/yqlib/doc/operators/reduce.md +++ /dev/null @@ -1,77 +0,0 @@ -# Reduce - -Reduce is a powerful way to process a collection of data into a new form. - -``` - as $ ireduce (; ) -``` - -e.g. - -``` -.[] as $item ireduce (0; . + $item) -``` - -On the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements. - -On the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. - -## yq vs jq syntax -Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`). - -To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added. - -## Sum numbers -Given a sample.yml file of: -```yaml -- 10 -- 2 -- 5 -- 3 -``` -then -```bash -yq '.[] as $item ireduce (0; . + $item)' sample.yml -``` -will output -```yaml -20 -``` - -## Merge all yaml files together -Given a sample.yml file of: -```yaml -a: cat -``` -And another sample another.yml file of: -```yaml -b: dog -``` -then -```bash -yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml -``` -will output -```yaml -a: cat -b: dog -``` - -## Convert an array to an object -Given a sample.yml file of: -```yaml -- name: Cathy - has: apples -- name: Bob - has: bananas -``` -then -```bash -yq '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml -``` -will output -```yaml -Cathy: apples -Bob: bananas -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/reverse.md b/external/yq/pkg/yqlib/doc/operators/reverse.md deleted file mode 100755 index 77f4037..0000000 --- a/external/yq/pkg/yqlib/doc/operators/reverse.md +++ /dev/null @@ -1,42 +0,0 @@ -# Reverse - -Reverses the order of the items in an array - -## Reverse -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq 'reverse' sample.yml -``` -will output -```yaml -- 3 -- 2 -- 1 -``` - -## Sort descending by string field -Use sort with reverse to sort in descending order. - -Given a sample.yml file of: -```yaml -- a: banana -- a: cat -- a: apple -``` -then -```bash -yq 'sort_by(.a) | reverse' sample.yml -``` -will output -```yaml -- a: cat -- a: banana -- a: apple -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/select.md b/external/yq/pkg/yqlib/doc/operators/select.md deleted file mode 100755 index 67b91f8..0000000 --- a/external/yq/pkg/yqlib/doc/operators/select.md +++ /dev/null @@ -1,139 +0,0 @@ -# Select - -Select is used to filter arrays and maps by a boolean expression. - -## Related Operators - -- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) -- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) -- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) - -## Select elements from array using wildcard prefix -Given a sample.yml file of: -```yaml -- cat -- goat -- dog -``` -then -```bash -yq '.[] | select(. == "*at")' sample.yml -``` -will output -```yaml -cat -goat -``` - -## Select elements from array using wildcard suffix -Given a sample.yml file of: -```yaml -- go-kart -- goat -- dog -``` -then -```bash -yq '.[] | select(. == "go*")' sample.yml -``` -will output -```yaml -go-kart -goat -``` - -## Select elements from array using wildcard prefix and suffix -Given a sample.yml file of: -```yaml -- ago -- go -- meow -- going -``` -then -```bash -yq '.[] | select(. == "*go*")' sample.yml -``` -will output -```yaml -ago -go -going -``` - -## Select elements from array with regular expression -See more regular expression examples under the `string` operator docs. - -Given a sample.yml file of: -```yaml -- this_0 -- not_this -- nor_0_this -- thisTo_4 -``` -then -```bash -yq '.[] | select(test("[a-zA-Z]+_[0-9]$"))' sample.yml -``` -will output -```yaml -this_0 -thisTo_4 -``` - -## Select items from a map -Given a sample.yml file of: -```yaml -things: cat -bob: goat -horse: dog -``` -then -```bash -yq '.[] | select(. == "cat" or test("og$"))' sample.yml -``` -will output -```yaml -cat -dog -``` - -## Use select and with_entries to filter map keys -Given a sample.yml file of: -```yaml -name: bob -legs: 2 -game: poker -``` -then -```bash -yq 'with_entries(select(.key | test("ame$")))' sample.yml -``` -will output -```yaml -name: bob -game: poker -``` - -## Select multiple items in a map and update -Note the brackets around the entire LHS. - -Given a sample.yml file of: -```yaml -a: - things: cat - bob: goat - horse: dog -``` -then -```bash -yq '(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"' sample.yml -``` -will output -```yaml -a: - things: rabbit - bob: rabbit - horse: dog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/slice-array.md b/external/yq/pkg/yqlib/doc/operators/slice-array.md deleted file mode 100755 index 40ebea4..0000000 --- a/external/yq/pkg/yqlib/doc/operators/slice-array.md +++ /dev/null @@ -1,105 +0,0 @@ -# Slice/Splice Array - -The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. - -You may leave out the first or second number, which will will refer to the start or end of the array respectively. - -## Slicing arrays -Given a sample.yml file of: -```yaml -- cat -- dog -- frog -- cow -``` -then -```bash -yq '.[1:3]' sample.yml -``` -will output -```yaml -- dog -- frog -``` - -## Slicing arrays - without the first number -Starts from the start of the array - -Given a sample.yml file of: -```yaml -- cat -- dog -- frog -- cow -``` -then -```bash -yq '.[:2]' sample.yml -``` -will output -```yaml -- cat -- dog -``` - -## Slicing arrays - without the second number -Finishes at the end of the array - -Given a sample.yml file of: -```yaml -- cat -- dog -- frog -- cow -``` -then -```bash -yq '.[2:]' sample.yml -``` -will output -```yaml -- frog -- cow -``` - -## Slicing arrays - use negative numbers to count backwards from the end -Given a sample.yml file of: -```yaml -- cat -- dog -- frog -- cow -``` -then -```bash -yq '.[1:-1]' sample.yml -``` -will output -```yaml -- dog -- frog -``` - -## Inserting into the middle of an array -using an expression to find the index - -Given a sample.yml file of: -```yaml -- cat -- dog -- frog -- cow -``` -then -```bash -yq '(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]' sample.yml -``` -will output -```yaml -- cat -- dog -- rabbit -- frog -- cow -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/sort-keys.md b/external/yq/pkg/yqlib/doc/operators/sort-keys.md deleted file mode 100755 index 41a4340..0000000 --- a/external/yq/pkg/yqlib/doc/operators/sort-keys.md +++ /dev/null @@ -1,75 +0,0 @@ -# Sort Keys - -The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). - -Sort is particularly useful for diffing two different yaml documents: - -```bash -yq -i -P 'sort_keys(..)' file1.yml -yq -i -P 'sort_keys(..)' file2.yml -diff file1.yml file2.yml -``` - -Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors. - -For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`. -See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. - -## Sort keys of map -Given a sample.yml file of: -```yaml -c: frog -a: blah -b: bing -``` -then -```bash -yq 'sort_keys(.)' sample.yml -``` -will output -```yaml -a: blah -b: bing -c: frog -``` - -## Sort keys recursively -Note the array elements are left unsorted, but maps inside arrays are sorted - -Given a sample.yml file of: -```yaml -bParent: - c: dog - array: - - 3 - - 1 - - 2 -aParent: - z: donkey - x: - - c: yum - b: delish - - b: ew - a: apple -``` -then -```bash -yq 'sort_keys(..)' sample.yml -``` -will output -```yaml -aParent: - x: - - b: delish - c: yum - - a: apple - b: ew - z: donkey -bParent: - array: - - 3 - - 1 - - 2 - c: dog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/sort.md b/external/yq/pkg/yqlib/doc/operators/sort.md deleted file mode 100755 index 51c79e4..0000000 --- a/external/yq/pkg/yqlib/doc/operators/sort.md +++ /dev/null @@ -1,181 +0,0 @@ -# Sort - -Sorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield). - -To sort by descending order, pipe the results through the `reverse` operator after sorting. - -Note that at this stage, `yq` only sorts scalar fields. - - -## Sort by string field -Given a sample.yml file of: -```yaml -- a: banana -- a: cat -- a: apple -``` -then -```bash -yq 'sort_by(.a)' sample.yml -``` -will output -```yaml -- a: apple -- a: banana -- a: cat -``` - -## Sort descending by string field -Use sort with reverse to sort in descending order. - -Given a sample.yml file of: -```yaml -- a: banana -- a: cat -- a: apple -``` -then -```bash -yq 'sort_by(.a) | reverse' sample.yml -``` -will output -```yaml -- a: cat -- a: banana -- a: apple -``` - -## Sort array in place -Given a sample.yml file of: -```yaml -cool: - - a: banana - - a: cat - - a: apple -``` -then -```bash -yq '.cool |= sort_by(.a)' sample.yml -``` -will output -```yaml -cool: - - a: apple - - a: banana - - a: cat -``` - -## Sort array of objects by key -Note that you can give sort_by complex expressions, not just paths - -Given a sample.yml file of: -```yaml -cool: - - b: banana - - a: banana - - c: banana -``` -then -```bash -yq '.cool |= sort_by(keys | .[0])' sample.yml -``` -will output -```yaml -cool: - - a: banana - - b: banana - - c: banana -``` - -## Sort is stable -Note the order of the elements in unchanged when equal in sorting. - -Given a sample.yml file of: -```yaml -- a: banana - b: 1 -- a: banana - b: 2 -- a: banana - b: 3 -- a: banana - b: 4 -``` -then -```bash -yq 'sort_by(.a)' sample.yml -``` -will output -```yaml -- a: banana - b: 1 -- a: banana - b: 2 -- a: banana - b: 3 -- a: banana - b: 4 -``` - -## Sort by numeric field -Given a sample.yml file of: -```yaml -- a: 10 -- a: 100 -- a: 1 -``` -then -```bash -yq 'sort_by(.a)' sample.yml -``` -will output -```yaml -- a: 1 -- a: 10 -- a: 100 -``` - -## Sort by custom date field -Given a sample.yml file of: -```yaml -- a: 12-Jun-2011 -- a: 23-Dec-2010 -- a: 10-Aug-2011 -``` -then -```bash -yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml -``` -will output -```yaml -- a: 23-Dec-2010 -- a: 12-Jun-2011 -- a: 10-Aug-2011 -``` - -## Sort, nulls come first -Given a sample.yml file of: -```yaml -- 8 -- 3 -- null -- 6 -- true -- false -- cat -``` -then -```bash -yq 'sort' sample.yml -``` -will output -```yaml -- null -- false -- true -- 3 -- 6 -- 8 -- cat -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/split-into-documents.md b/external/yq/pkg/yqlib/doc/operators/split-into-documents.md deleted file mode 100755 index e072328..0000000 --- a/external/yq/pkg/yqlib/doc/operators/split-into-documents.md +++ /dev/null @@ -1,31 +0,0 @@ -# Split into Documents - -This operator splits all matches into separate documents - -## Split empty -Running -```bash -yq --null-input 'split_doc' -``` -will output -```yaml - -``` - -## Split array -Given a sample.yml file of: -```yaml -- a: cat -- b: dog -``` -then -```bash -yq '.[] | split_doc' sample.yml -``` -will output -```yaml -a: cat ---- -b: dog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/string-operators.md b/external/yq/pkg/yqlib/doc/operators/string-operators.md deleted file mode 100755 index 585404c..0000000 --- a/external/yq/pkg/yqlib/doc/operators/string-operators.md +++ /dev/null @@ -1,385 +0,0 @@ -# String Operators - -## RegEx -This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax. - -Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`. - -### match(regEx) -This operator returns the substring match details of the given regEx. - -### capture(regEx) -Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing. - -## test(regEx) -Returns true if the string matches the RegEx, false otherwise. - -## sub(regEx, replacement) -Substitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second is a what to replace those matches with. This can refer to capture groups from the first RegEx. - -## String blocks, bash and newlines -Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_. - -For instance to get this yaml: - -``` -a: | - cat -``` - -Using `$( exp )` wont work, as it will trim the trailing new line. - -``` -m=$(echo "cat\n") yq -n '.a = strenv(m)' -a: cat -``` - -However, using printf works: -``` -printf -v m "cat\n" ; m="$m" yq -n '.a = strenv(m)' -a: | - cat -``` - -As well as having multiline expressions: -``` -m="cat -" yq -n '.a = strenv(m)' -a: | - cat -``` - -Similarly, if you're trying to set the content from a file, and want a trailing new line: - -``` -IFS= read -rd '' output < <(cat my_file) -output=$output ./yq '.data.values = strenv(output)' first.yml -``` - -## To up (upper) case -Works with unicode characters - -Given a sample.yml file of: -```yaml -água -``` -then -```bash -yq 'upcase' sample.yml -``` -will output -```yaml -ÁGUA -``` - -## To down (lower) case -Works with unicode characters - -Given a sample.yml file of: -```yaml -ÁgUA -``` -then -```bash -yq 'downcase' sample.yml -``` -will output -```yaml -água -``` - -## Join strings -Given a sample.yml file of: -```yaml -- cat -- meow -- 1 -- null -- true -``` -then -```bash -yq 'join("; ")' sample.yml -``` -will output -```yaml -cat; meow; 1; ; true -``` - -## Trim strings -Given a sample.yml file of: -```yaml -- ' cat' -- 'dog ' -- ' cow cow ' -- horse -``` -then -```bash -yq '.[] | trim' sample.yml -``` -will output -```yaml -cat -dog -cow cow -horse -``` - -## Match string -Given a sample.yml file of: -```yaml -foo bar foo -``` -then -```bash -yq 'match("foo")' sample.yml -``` -will output -```yaml -string: foo -offset: 0 -length: 3 -captures: [] -``` - -## Match string, case insensitive -Given a sample.yml file of: -```yaml -foo bar FOO -``` -then -```bash -yq '[match("(?i)foo"; "g")]' sample.yml -``` -will output -```yaml -- string: foo - offset: 0 - length: 3 - captures: [] -- string: FOO - offset: 8 - length: 3 - captures: [] -``` - -## Match with global capture group -Given a sample.yml file of: -```yaml -abc abc -``` -then -```bash -yq '[match("(ab)(c)"; "g")]' sample.yml -``` -will output -```yaml -- string: abc - offset: 0 - length: 3 - captures: - - string: ab - offset: 0 - length: 2 - - string: c - offset: 2 - length: 1 -- string: abc - offset: 4 - length: 3 - captures: - - string: ab - offset: 4 - length: 2 - - string: c - offset: 6 - length: 1 -``` - -## Match with named capture groups -Given a sample.yml file of: -```yaml -foo bar foo foo foo -``` -then -```bash -yq '[match("foo (?Pbar)? foo"; "g")]' sample.yml -``` -will output -```yaml -- string: foo bar foo - offset: 0 - length: 11 - captures: - - string: bar - offset: 4 - length: 3 - name: bar123 -- string: foo foo - offset: 12 - length: 8 - captures: - - string: null - offset: -1 - length: 0 - name: bar123 -``` - -## Capture named groups into a map -Given a sample.yml file of: -```yaml -xyzzy-14 -``` -then -```bash -yq 'capture("(?P[a-z]+)-(?P[0-9]+)")' sample.yml -``` -will output -```yaml -a: xyzzy -n: "14" -``` - -## Match without global flag -Given a sample.yml file of: -```yaml -cat cat -``` -then -```bash -yq 'match("cat")' sample.yml -``` -will output -```yaml -string: cat -offset: 0 -length: 3 -captures: [] -``` - -## Match with global flag -Given a sample.yml file of: -```yaml -cat cat -``` -then -```bash -yq '[match("cat"; "g")]' sample.yml -``` -will output -```yaml -- string: cat - offset: 0 - length: 3 - captures: [] -- string: cat - offset: 4 - length: 3 - captures: [] -``` - -## Test using regex -Like jq's equivalent, this works like match but only returns true/false instead of full match details - -Given a sample.yml file of: -```yaml -- cat -- dog -``` -then -```bash -yq '.[] | test("at")' sample.yml -``` -will output -```yaml -true -false -``` - -## Substitute / Replace string -This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax) -Note the use of `|=` to run in context of the current string value. - -Given a sample.yml file of: -```yaml -a: dogs are great -``` -then -```bash -yq '.a |= sub("dogs", "cats")' sample.yml -``` -will output -```yaml -a: cats are great -``` - -## Substitute / Replace string with regex -This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax) -Note the use of `|=` to run in context of the current string value. - -Given a sample.yml file of: -```yaml -a: cat -b: heat -``` -then -```bash -yq '.[] |= sub("(a)", "${1}r")' sample.yml -``` -will output -```yaml -a: cart -b: heart -``` - -## Custom types: that are really strings -When custom tags are encountered, yq will try to decode the underlying type. - -Given a sample.yml file of: -```yaml -a: !horse cat -b: !goat heat -``` -then -```bash -yq '.[] |= sub("(a)", "${1}r")' sample.yml -``` -will output -```yaml -a: !horse cart -b: !goat heart -``` - -## Split strings -Given a sample.yml file of: -```yaml -cat; meow; 1; ; true -``` -then -```bash -yq 'split("; ")' sample.yml -``` -will output -```yaml -- cat -- meow -- "1" -- "" -- "true" -``` - -## Split strings one match -Given a sample.yml file of: -```yaml -word -``` -then -```bash -yq 'split("; ")' sample.yml -``` -will output -```yaml -- word -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/style.md b/external/yq/pkg/yqlib/doc/operators/style.md deleted file mode 100755 index b21a5b2..0000000 --- a/external/yq/pkg/yqlib/doc/operators/style.md +++ /dev/null @@ -1,240 +0,0 @@ -# Style - -The style operator can be used to get or set the style of nodes (e.g. string style, yaml style) - -## Update and set style of a particular node (simple) -Given a sample.yml file of: -```yaml -a: - b: thing - c: something -``` -then -```bash -yq '.a.b = "new" | .a.b style="double"' sample.yml -``` -will output -```yaml -a: - b: "new" - c: something -``` - -## Update and set style of a particular node using path variables -Given a sample.yml file of: -```yaml -a: - b: thing - c: something -``` -then -```bash -yq 'with(.a.b ; . = "new" | . style="double")' sample.yml -``` -will output -```yaml -a: - b: "new" - c: something -``` - -## Set tagged style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="tagged"' sample.yml -``` -will output -```yaml -!!map -a: !!str cat -b: !!int 5 -c: !!float 3.2 -e: !!bool true -``` - -## Set double quote style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="double"' sample.yml -``` -will output -```yaml -a: "cat" -b: "5" -c: "3.2" -e: "true" -``` - -## Set double quote style on map keys too -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '... style="double"' sample.yml -``` -will output -```yaml -"a": "cat" -"b": "5" -"c": "3.2" -"e": "true" -``` - -## Set single quote style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="single"' sample.yml -``` -will output -```yaml -a: 'cat' -b: '5' -c: '3.2' -e: 'true' -``` - -## Set literal quote style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="literal"' sample.yml -``` -will output -```yaml -a: |- - cat -b: |- - 5 -c: |- - 3.2 -e: |- - true -``` - -## Set folded quote style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="folded"' sample.yml -``` -will output -```yaml -a: >- - cat -b: >- - 5 -c: >- - 3.2 -e: >- - true -``` - -## Set flow quote style -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '.. style="flow"' sample.yml -``` -will output -```yaml -{a: cat, b: 5, c: 3.2, e: true} -``` - -## Reset style - or pretty print -Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this. - -Given a sample.yml file of: -```yaml -a: cat -"b": 5 -'c': 3.2 -"e": true -``` -then -```bash -yq '... style=""' sample.yml -``` -will output -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` - -## Set style relatively with assign-update -Given a sample.yml file of: -```yaml -a: single -b: double -``` -then -```bash -yq '.[] style |= .' sample.yml -``` -will output -```yaml -a: 'single' -b: "double" -``` - -## Read style -Given a sample.yml file of: -```yaml -{a: "cat", b: 'thing'} -``` -then -```bash -yq '.. | style' sample.yml -``` -will output -```yaml -flow -double -single -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/subtract.md b/external/yq/pkg/yqlib/doc/operators/subtract.md deleted file mode 100755 index b850cc9..0000000 --- a/external/yq/pkg/yqlib/doc/operators/subtract.md +++ /dev/null @@ -1,163 +0,0 @@ -# Subtract - -You can use subtract to subtract numbers, as well as removing elements from an array. - -## Array subtraction -Running -```bash -yq --null-input '[1,2] - [2,3]' -``` -will output -```yaml -- 1 -``` - -## Array subtraction with nested array -Running -```bash -yq --null-input '[[1], 1, 2] - [[1], 3]' -``` -will output -```yaml -- 1 -- 2 -``` - -## Array subtraction with nested object -Note that order of the keys does not matter - -Given a sample.yml file of: -```yaml -- a: b - c: d -- a: b -``` -then -```bash -yq '. - [{"c": "d", "a": "b"}]' sample.yml -``` -will output -```yaml -- a: b -``` - -## Number subtraction - float -If the lhs or rhs are floats then the expression will be calculated with floats. - -Given a sample.yml file of: -```yaml -a: 3 -b: 4.5 -``` -then -```bash -yq '.a = .a - .b' sample.yml -``` -will output -```yaml -a: -1.5 -b: 4.5 -``` - -## Number subtraction - float -If the lhs or rhs are floats then the expression will be calculated with floats. - -Given a sample.yml file of: -```yaml -a: 3 -b: 4.5 -``` -then -```bash -yq '.a = .a - .b' sample.yml -``` -will output -```yaml -a: -1.5 -b: 4.5 -``` - -## Number subtraction - int -If both the lhs and rhs are ints then the expression will be calculated with ints. - -Given a sample.yml file of: -```yaml -a: 3 -b: 4 -``` -then -```bash -yq '.a = .a - .b' sample.yml -``` -will output -```yaml -a: -1 -b: 4 -``` - -## Decrement numbers -Given a sample.yml file of: -```yaml -a: 3 -b: 5 -``` -then -```bash -yq '.[] -= 1' sample.yml -``` -will output -```yaml -a: 2 -b: 4 -``` - -## Date subtraction -You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. - -Given a sample.yml file of: -```yaml -a: 2021-01-01T03:10:00Z -``` -then -```bash -yq '.a -= "3h10m"' sample.yml -``` -will output -```yaml -a: 2021-01-01T00:00:00Z -``` - -## Date subtraction - custom format -Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. - -Given a sample.yml file of: -```yaml -a: Saturday, 15-Dec-01 at 6:00AM GMT -``` -then -```bash -yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml -``` -will output -```yaml -a: Saturday, 15-Dec-01 at 2:59AM GMT -``` - -## Custom types: that are really numbers -When custom tags are encountered, yq will try to decode the underlying type. - -Given a sample.yml file of: -```yaml -a: !horse 2 -b: !goat 1 -``` -then -```bash -yq '.a -= .b' sample.yml -``` -will output -```yaml -a: !horse 1 -b: !goat 1 -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/tag.md b/external/yq/pkg/yqlib/doc/operators/tag.md deleted file mode 100755 index 43db143..0000000 --- a/external/yq/pkg/yqlib/doc/operators/tag.md +++ /dev/null @@ -1,84 +0,0 @@ -# Tag - -The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`). - -## Get tag -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -f: [] -``` -then -```bash -yq '.. | tag' sample.yml -``` -will output -```yaml -!!map -!!str -!!int -!!float -!!bool -!!seq -``` - -## type is an alias for tag -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -f: [] -``` -then -```bash -yq '.. | type' sample.yml -``` -will output -```yaml -!!map -!!str -!!int -!!float -!!bool -!!seq -``` - -## Set custom tag -Given a sample.yml file of: -```yaml -a: str -``` -then -```bash -yq '.a tag = "!!mikefarah"' sample.yml -``` -will output -```yaml -a: !!mikefarah str -``` - -## Find numbers and convert them to strings -Given a sample.yml file of: -```yaml -a: cat -b: 5 -c: 3.2 -e: true -``` -then -```bash -yq '(.. | select(tag == "!!int")) tag= "!!str"' sample.yml -``` -will output -```yaml -a: cat -b: "5" -c: 3.2 -e: true -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/traverse-read.md b/external/yq/pkg/yqlib/doc/operators/traverse-read.md deleted file mode 100755 index 3a4452e..0000000 --- a/external/yq/pkg/yqlib/doc/operators/traverse-read.md +++ /dev/null @@ -1,485 +0,0 @@ -# Traverse (Read) - -This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures. - -## Simple map navigation -Given a sample.yml file of: -```yaml -a: - b: apple -``` -then -```bash -yq '.a' sample.yml -``` -will output -```yaml -b: apple -``` - -## Splat -Often used to pipe children into other operators - -Given a sample.yml file of: -```yaml -- b: apple -- c: banana -``` -then -```bash -yq '.[]' sample.yml -``` -will output -```yaml -b: apple -c: banana -``` - -## Optional Splat -Just like splat, but won't error if you run it against scalars - -Given a sample.yml file of: -```yaml -cat -``` -then -```bash -yq '.[]' sample.yml -``` -will output -```yaml -``` - -## Special characters -Use quotes with brackets around path elements with special characters - -Given a sample.yml file of: -```yaml -"{}": frog -``` -then -```bash -yq '.["{}"]' sample.yml -``` -will output -```yaml -frog -``` - -## Nested special characters -Given a sample.yml file of: -```yaml -a: - "key.withdots": - "another.key": apple -``` -then -```bash -yq '.a["key.withdots"]["another.key"]' sample.yml -``` -will output -```yaml -apple -``` - -## Keys with spaces -Use quotes with brackets around path elements with special characters - -Given a sample.yml file of: -```yaml -"red rabbit": frog -``` -then -```bash -yq '.["red rabbit"]' sample.yml -``` -will output -```yaml -frog -``` - -## Dynamic keys -Expressions within [] can be used to dynamically lookup / calculate keys - -Given a sample.yml file of: -```yaml -b: apple -apple: crispy yum -banana: soft yum -``` -then -```bash -yq '.[.b]' sample.yml -``` -will output -```yaml -crispy yum -``` - -## Children don't exist -Nodes are added dynamically while traversing - -Given a sample.yml file of: -```yaml -c: banana -``` -then -```bash -yq '.a.b' sample.yml -``` -will output -```yaml -null -``` - -## Optional identifier -Like jq, does not output an error when the yaml is not an array or object as expected - -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq '.a?' sample.yml -``` -will output -```yaml -``` - -## Wildcard matching -Given a sample.yml file of: -```yaml -a: - cat: apple - mad: things -``` -then -```bash -yq '.a."*a*"' sample.yml -``` -will output -```yaml -apple -things -``` - -## Aliases -Given a sample.yml file of: -```yaml -a: &cat - c: frog -b: *cat -``` -then -```bash -yq '.b' sample.yml -``` -will output -```yaml -*cat -``` - -## Traversing aliases with splat -Given a sample.yml file of: -```yaml -a: &cat - c: frog -b: *cat -``` -then -```bash -yq '.b[]' sample.yml -``` -will output -```yaml -frog -``` - -## Traversing aliases explicitly -Given a sample.yml file of: -```yaml -a: &cat - c: frog -b: *cat -``` -then -```bash -yq '.b.c' sample.yml -``` -will output -```yaml -frog -``` - -## Traversing arrays by index -Given a sample.yml file of: -```yaml -- 1 -- 2 -- 3 -``` -then -```bash -yq '.[0]' sample.yml -``` -will output -```yaml -1 -``` - -## Traversing nested arrays by index -Given a sample.yml file of: -```yaml -[[], [cat]] -``` -then -```bash -yq '.[1][0]' sample.yml -``` -will output -```yaml -cat -``` - -## Maps with numeric keys -Given a sample.yml file of: -```yaml -2: cat -``` -then -```bash -yq '.[2]' sample.yml -``` -will output -```yaml -cat -``` - -## Maps with non existing numeric keys -Given a sample.yml file of: -```yaml -a: b -``` -then -```bash -yq '.[0]' sample.yml -``` -will output -```yaml -null -``` - -## Traversing merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar.a' sample.yml -``` -will output -```yaml -foo_a -``` - -## Traversing merge anchors with override -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar.c' sample.yml -``` -will output -```yaml -foo_c -``` - -## Traversing merge anchors with local override -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar.thing' sample.yml -``` -will output -```yaml -foobar_thing -``` - -## Splatting merge anchors -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobar[]' sample.yml -``` -will output -```yaml -foo_c -foo_a -foobar_thing -``` - -## Traversing merge anchor lists -Note that the later merge anchors override previous - -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobarList.thing' sample.yml -``` -will output -```yaml -bar_thing -``` - -## Splatting merge anchor lists -Given a sample.yml file of: -```yaml -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: foobarList_b - !!merge <<: - - *foo - - *bar - c: foobarList_c -foobar: - c: foobar_c - !!merge <<: *foo - thing: foobar_thing -``` -then -```bash -yq '.foobarList[]' sample.yml -``` -will output -```yaml -bar_b -foo_a -bar_thing -foobarList_c -``` - -## Select multiple indices -Given a sample.yml file of: -```yaml -a: - - a - - b - - c -``` -then -```bash -yq '.a[0, 2]' sample.yml -``` -will output -```yaml -a -c -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/union.md b/external/yq/pkg/yqlib/doc/operators/union.md deleted file mode 100755 index abf0bef..0000000 --- a/external/yq/pkg/yqlib/doc/operators/union.md +++ /dev/null @@ -1,33 +0,0 @@ -# Union - -This operator is used to combine different results together. - -## Combine scalars -Running -```bash -yq --null-input '1, true, "cat"' -``` -will output -```yaml -1 -true -cat -``` - -## Combine selected paths -Given a sample.yml file of: -```yaml -a: fieldA -b: fieldB -c: fieldC -``` -then -```bash -yq '.a, .c' sample.yml -``` -will output -```yaml -fieldA -fieldC -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/unique.md b/external/yq/pkg/yqlib/doc/operators/unique.md deleted file mode 100755 index a1bc443..0000000 --- a/external/yq/pkg/yqlib/doc/operators/unique.md +++ /dev/null @@ -1,87 +0,0 @@ -# Unique - -This is used to filter out duplicated items in an array. Note that the original order of the array is maintained. - - -## Unique array of scalars (string/numbers) -Note that unique maintains the original order of the array. - -Given a sample.yml file of: -```yaml -- 2 -- 1 -- 3 -- 2 -``` -then -```bash -yq 'unique' sample.yml -``` -will output -```yaml -- 2 -- 1 -- 3 -``` - -## Unique nulls -Unique works on the node value, so it considers different representations of nulls to be different - -Given a sample.yml file of: -```yaml -- ~ -- null -- ~ -- null -``` -then -```bash -yq 'unique' sample.yml -``` -will output -```yaml -- ~ -- null -``` - -## Unique all nulls -Run against the node tag to unique all the nulls - -Given a sample.yml file of: -```yaml -- ~ -- null -- ~ -- null -``` -then -```bash -yq 'unique_by(tag)' sample.yml -``` -will output -```yaml -- ~ -``` - -## Unique array object fields -Given a sample.yml file of: -```yaml -- name: harry - pet: cat -- name: billy - pet: dog -- name: harry - pet: dog -``` -then -```bash -yq 'unique_by(.name)' sample.yml -``` -will output -```yaml -- name: harry - pet: cat -- name: billy - pet: dog -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/variable-operators.md b/external/yq/pkg/yqlib/doc/operators/variable-operators.md deleted file mode 100755 index 5c9a293..0000000 --- a/external/yq/pkg/yqlib/doc/operators/variable-operators.md +++ /dev/null @@ -1,98 +0,0 @@ -# Variable Operators - -Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). - -Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. - -## Single value variable -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq '.a as $foo | $foo' sample.yml -``` -will output -```yaml -cat -``` - -## Multi value variable -Given a sample.yml file of: -```yaml -- cat -- dog -``` -then -```bash -yq '.[] as $foo | $foo' sample.yml -``` -will output -```yaml -cat -dog -``` - -## Using variables as a lookup -Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...) - -Given a sample.yml file of: -```yaml -"posts": - - "title": First post - "author": anon - - "title": A well-written article - "author": person1 -"realnames": - "anon": Anonymous Coward - "person1": Person McPherson -``` -then -```bash -yq '.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}' sample.yml -``` -will output -```yaml -title: First post -author: Anonymous Coward -title: A well-written article -author: Person McPherson -``` - -## Using variables to swap values -Given a sample.yml file of: -```yaml -a: a_value -b: b_value -``` -then -```bash -yq '.a as $x | .b as $y | .b = $x | .a = $y' sample.yml -``` -will output -```yaml -a: b_value -b: a_value -``` - -## Use ref to reference a path repeatedly -Note: You may find the `with` operator more useful. - -Given a sample.yml file of: -```yaml -a: - b: thing - c: something -``` -then -```bash -yq '.a.b ref $x | $x = "new" | $x style="double"' sample.yml -``` -will output -```yaml -a: - b: "new" - c: something -``` - diff --git a/external/yq/pkg/yqlib/doc/operators/with.md b/external/yq/pkg/yqlib/doc/operators/with.md deleted file mode 100755 index 6b1a913..0000000 --- a/external/yq/pkg/yqlib/doc/operators/with.md +++ /dev/null @@ -1,64 +0,0 @@ -# With - -Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context. - -## Update and style -Given a sample.yml file of: -```yaml -a: - deeply: - nested: value -``` -then -```bash -yq 'with(.a.deeply.nested; . = "newValue" | . style="single")' sample.yml -``` -will output -```yaml -a: - deeply: - nested: 'newValue' -``` - -## Update multiple deeply nested properties -Given a sample.yml file of: -```yaml -a: - deeply: - nested: value - other: thing -``` -then -```bash -yq 'with(.a.deeply; .nested = "newValue" | .other= "newThing")' sample.yml -``` -will output -```yaml -a: - deeply: - nested: newValue - other: newThing -``` - -## Update array elements relatively -The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element. - -Given a sample.yml file of: -```yaml -myArray: - - a: apple - - a: banana -``` -then -```bash -yq 'with(.myArray[]; .b = .a + " yum")' sample.yml -``` -will output -```yaml -myArray: - - a: apple - b: apple yum - - a: banana - b: banana yum -``` - diff --git a/external/yq/pkg/yqlib/doc/usage/convert.md b/external/yq/pkg/yqlib/doc/usage/convert.md deleted file mode 100755 index 1f149c1..0000000 --- a/external/yq/pkg/yqlib/doc/usage/convert.md +++ /dev/null @@ -1,253 +0,0 @@ -# JSON - -Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON). - -Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. - - -## Parse json: simple -JSON is a subset of yaml, so all you need to do is prettify the output - -Given a sample.json file of: -```json -{"cat": "meow"} -``` -then -```bash -yq -P '.' sample.json -``` -will output -```yaml -cat: meow -``` - -## Parse json: complex -JSON is a subset of yaml, so all you need to do is prettify the output - -Given a sample.json file of: -```json -{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}} -``` -then -```bash -yq -P '.' sample.json -``` -will output -```yaml -a: Easy! as one two three -b: - c: 2 - d: - - 3 - - 4 -``` - -## Encode json: simple -Given a sample.yml file of: -```yaml -cat: meow -``` -then -```bash -yq -o=json '.' sample.yml -``` -will output -```json -{ - "cat": "meow" -} -``` - -## Encode json: simple - in one line -Given a sample.yml file of: -```yaml -cat: meow # this is a comment, and it will be dropped. -``` -then -```bash -yq -o=json -I=0 '.' sample.yml -``` -will output -```json -{"cat":"meow"} -``` - -## Encode json: comments -Given a sample.yml file of: -```yaml -cat: meow # this is a comment, and it will be dropped. -``` -then -```bash -yq -o=json '.' sample.yml -``` -will output -```json -{ - "cat": "meow" -} -``` - -## Encode json: anchors -Anchors are dereferenced - -Given a sample.yml file of: -```yaml -cat: &ref meow -anotherCat: *ref -``` -then -```bash -yq -o=json '.' sample.yml -``` -will output -```json -{ - "cat": "meow", - "anotherCat": "meow" -} -``` - -## Encode json: multiple results -Each matching node is converted into a json doc. This is best used with 0 indent (json document per line) - -Given a sample.yml file of: -```yaml -things: [{stuff: cool}, {whatever: cat}] -``` -then -```bash -yq -o=json -I=0 '.things[]' sample.yml -``` -will output -```json -{"stuff":"cool"} -{"whatever":"cat"} -``` - -## Roundtrip NDJSON -Unfortunately the json encoder strips leading spaces of values. - -Given a sample.json file of: -```json -{"this": "is a multidoc json file"} -{"each": ["line is a valid json document"]} -{"a number": 4} - -``` -then -```bash -yq -p=json -o=json -I=0 sample.json -``` -will output -```yaml -{"this":"is a multidoc json file"} -{"each":["line is a valid json document"]} -{"a number":4} -``` - -## Roundtrip multi-document JSON -The NDJSON parser can also handle multiple multi-line json documents in a single file! - -Given a sample.json file of: -```json -{ - "this": "is a multidoc json file" -} -{ - "it": [ - "has", - "consecutive", - "json documents" - ] -} -{ - "a number": 4 -} - -``` -then -```bash -yq -p=json -o=json -I=2 sample.json -``` -will output -```yaml -{ - "this": "is a multidoc json file" -} -{ - "it": [ - "has", - "consecutive", - "json documents" - ] -} -{ - "a number": 4 -} -``` - -## Update a specific document in a multi-document json -Documents are indexed by the `documentIndex` or `di` operator. - -Given a sample.json file of: -```json -{"this": "is a multidoc json file"} -{"each": ["line is a valid json document"]} -{"a number": 4} - -``` -then -```bash -yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json -``` -will output -```yaml -{"this":"is a multidoc json file"} -{"each":["line is a valid json document","cool"]} -{"a number":4} -``` - -## Find and update a specific document in a multi-document json -Use expressions as you normally would. - -Given a sample.json file of: -```json -{"this": "is a multidoc json file"} -{"each": ["line is a valid json document"]} -{"a number": 4} - -``` -then -```bash -yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json -``` -will output -```yaml -{"this":"is a multidoc json file"} -{"each":["line is a valid json document","cool"]} -{"a number":4} -``` - -## Decode NDJSON -Given a sample.json file of: -```json -{"this": "is a multidoc json file"} -{"each": ["line is a valid json document"]} -{"a number": 4} - -``` -then -```bash -yq -p=json sample.json -``` -will output -```yaml -this: is a multidoc json file ---- -each: - - line is a valid json document ---- -a number: 4 -``` - diff --git a/external/yq/pkg/yqlib/doc/usage/csv-tsv.md b/external/yq/pkg/yqlib/doc/usage/csv-tsv.md deleted file mode 100755 index c58e42c..0000000 --- a/external/yq/pkg/yqlib/doc/usage/csv-tsv.md +++ /dev/null @@ -1,208 +0,0 @@ -# CSV -Encode/Decode/Roundtrip CSV and TSV files. - -## Encode -Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: - -```yaml -- name: Bobo - type: dog -- name: Fifi - type: cat -``` - -As well as arrays of arrays of scalars (strings/numbers/booleans): - -```yaml -- [Bobo, dog] -- [Fifi, cat] -``` - -## Decode -Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries. -The data will be coded into an array of objects, using the header rows as keys. - -```csv -name,type -Bobo,dog -Fifi,cat -``` - - -## Encode CSV simple -Given a sample.yml file of: -```yaml -- [i, like, csv] -- [because, excel, is, cool] -``` -then -```bash -yq -o=csv sample.yml -``` -will output -```csv -i,like,csv -because,excel,is,cool -``` - -## Encode TSV simple -Given a sample.yml file of: -```yaml -- [i, like, csv] -- [because, excel, is, cool] -``` -then -```bash -yq -o=tsv sample.yml -``` -will output -```tsv -i like csv -because excel is cool -``` - -## Encode array of objects to csv -Given a sample.yml file of: -```yaml -- name: Gary - numberOfCats: 1 - likesApples: true - height: 168.8 -- name: Samantha's Rabbit - numberOfCats: 2 - likesApples: false - height: -188.8 - -``` -then -```bash -yq -o=csv sample.yml -``` -will output -```csv -name,numberOfCats,likesApples,height -Gary,1,true,168.8 -Samantha's Rabbit,2,false,-188.8 -``` - -## Encode array of objects to custom csv format -Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like. - -Given a sample.yml file of: -```yaml -- name: Gary - numberOfCats: 1 - likesApples: true - height: 168.8 -- name: Samantha's Rabbit - numberOfCats: 2 - likesApples: false - height: -188.8 - -``` -then -```bash -yq -o=csv '[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]' sample.yml -``` -will output -```csv -Name,Number of Cats -Gary,1 -Samantha's Rabbit,2 -``` - -## Encode array of objects to csv - missing fields behaviour -First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank - -Given a sample.yml file of: -```yaml -- name: Gary - numberOfCats: 1 - height: 168.8 -- name: Samantha's Rabbit - height: -188.8 - likesApples: false - -``` -then -```bash -yq -o=csv sample.yml -``` -will output -```csv -name,numberOfCats,height -Gary,1,168.8 -Samantha's Rabbit,,-188.8 -``` - -## Parse CSV into an array of objects -First row is assumed to be the header row. - -Given a sample.csv file of: -```csv -name,numberOfCats,likesApples,height -Gary,1,true,168.8 -Samantha's Rabbit,2,false,-188.8 - -``` -then -```bash -yq -p=csv sample.csv -``` -will output -```yaml -- name: Gary - numberOfCats: 1 - likesApples: true - height: 168.8 -- name: Samantha's Rabbit - numberOfCats: 2 - likesApples: false - height: -188.8 -``` - -## Parse TSV into an array of objects -First row is assumed to be the header row. - -Given a sample.tsv file of: -```tsv -name numberOfCats likesApples height -Gary 1 true 168.8 -Samantha's Rabbit 2 false -188.8 - -``` -then -```bash -yq -p=tsv sample.tsv -``` -will output -```yaml -- name: Gary - numberOfCats: 1 - likesApples: true - height: 168.8 -- name: Samantha's Rabbit - numberOfCats: 2 - likesApples: false - height: -188.8 -``` - -## Round trip -Given a sample.csv file of: -```csv -name,numberOfCats,likesApples,height -Gary,1,true,168.8 -Samantha's Rabbit,2,false,-188.8 - -``` -then -```bash -yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv -``` -will output -```csv -name,numberOfCats,likesApples,height -Gary,3,true,168.8 -Samantha's Rabbit,2,false,-188.8 -``` - diff --git a/external/yq/pkg/yqlib/doc/usage/headers/convert.md b/external/yq/pkg/yqlib/doc/usage/headers/convert.md deleted file mode 100755 index 0d852d9..0000000 --- a/external/yq/pkg/yqlib/doc/usage/headers/convert.md +++ /dev/null @@ -1,6 +0,0 @@ -# JSON - -Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON). - -Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. - diff --git a/external/yq/pkg/yqlib/doc/usage/headers/csv-tsv.md b/external/yq/pkg/yqlib/doc/usage/headers/csv-tsv.md deleted file mode 100755 index 0a7c7ca..0000000 --- a/external/yq/pkg/yqlib/doc/usage/headers/csv-tsv.md +++ /dev/null @@ -1,30 +0,0 @@ -# CSV -Encode/Decode/Roundtrip CSV and TSV files. - -## Encode -Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: - -```yaml -- name: Bobo - type: dog -- name: Fifi - type: cat -``` - -As well as arrays of arrays of scalars (strings/numbers/booleans): - -```yaml -- [Bobo, dog] -- [Fifi, cat] -``` - -## Decode -Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries. -The data will be coded into an array of objects, using the header rows as keys. - -```csv -name,type -Bobo,dog -Fifi,cat -``` - diff --git a/external/yq/pkg/yqlib/doc/usage/headers/properties.md b/external/yq/pkg/yqlib/doc/usage/headers/properties.md deleted file mode 100755 index 90164a6..0000000 --- a/external/yq/pkg/yqlib/doc/usage/headers/properties.md +++ /dev/null @@ -1,5 +0,0 @@ -# Properties - -Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across. - -By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these. diff --git a/external/yq/pkg/yqlib/doc/usage/headers/xml.md b/external/yq/pkg/yqlib/doc/usage/headers/xml.md deleted file mode 100755 index bcd9ba4..0000000 --- a/external/yq/pkg/yqlib/doc/usage/headers/xml.md +++ /dev/null @@ -1,40 +0,0 @@ -# XML - -Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are. - -Consecutive xml nodes with the same name are assumed to be arrays. - -XML content data, attributes processing instructions and directives are all created as plain fields. - -This can be controlled by: - -| Flag | Default |Sample XML | -| -- | -- | -- | - | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | - | `--xml-content-name` | `+content` | Meow in ```Meow true``` | - | `--xml-directive-name` | `+directive` | `````` | - | `--xml-proc-inst-prefix` | `+p_` | `````` | - - -{% hint style="warning" %} -Default Attribute Prefix will be changing in v4.30! -In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@". - -This will affect users that have not set their own prefix and are not roundtripping XML changes. - -{% endhint %} - -## Encoder / Decoder flag options - -In addition to the above flags, there are the following xml encoder/decoder options controlled by flags: - -| Flag | Default | Description | -| -- | -- | -- | -| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| -| `--xml-keep-namespace` | true | Keeps the namespace of attributes | -| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. | -| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` | -| `--xml-skip-directives` | false | Skips over directives, e.g. `````` | - - -See below for examples diff --git a/external/yq/pkg/yqlib/doc/usage/properties.md b/external/yq/pkg/yqlib/doc/usage/properties.md deleted file mode 100755 index f3d0123..0000000 --- a/external/yq/pkg/yqlib/doc/usage/properties.md +++ /dev/null @@ -1,195 +0,0 @@ -# Properties - -Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across. - -By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these. - -## Encode properties -Note that empty arrays and maps are not encoded by default. - -Given a sample.yml file of: -```yaml -# block comments come through -person: # neither do comments on maps - name: Mike Wazowski # comments on values appear - pets: - - cat # comments on array values appear - food: [pizza] # comments on arrays do not -emptyArray: [] -emptyMap: [] - -``` -then -```bash -yq -o=props sample.yml -``` -will output -```properties -# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -``` - -## Encode properties: scalar encapsulation -Note that string values with blank characters in them are encapsulated with double quotes - -Given a sample.yml file of: -```yaml -# block comments come through -person: # neither do comments on maps - name: Mike Wazowski # comments on values appear - pets: - - cat # comments on array values appear - food: [pizza] # comments on arrays do not -emptyArray: [] -emptyMap: [] - -``` -then -```bash -yq -o=props --unwrapScalar=false sample.yml -``` -will output -```properties -# block comments come through -# comments on values appear -person.name = "Mike Wazowski" - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -``` - -## Encode properties: no comments -Given a sample.yml file of: -```yaml -# block comments come through -person: # neither do comments on maps - name: Mike Wazowski # comments on values appear - pets: - - cat # comments on array values appear - food: [pizza] # comments on arrays do not -emptyArray: [] -emptyMap: [] - -``` -then -```bash -yq -o=props '... comments = ""' sample.yml -``` -will output -```properties -person.name = Mike Wazowski -person.pets.0 = cat -person.food.0 = pizza -``` - -## Encode properties: include empty maps and arrays -Use a yq expression to set the empty maps and sequences to your desired value. - -Given a sample.yml file of: -```yaml -# block comments come through -person: # neither do comments on maps - name: Mike Wazowski # comments on values appear - pets: - - cat # comments on array values appear - food: [pizza] # comments on arrays do not -emptyArray: [] -emptyMap: [] - -``` -then -```bash -yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ""' sample.yml -``` -will output -```properties -# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -emptyArray = -emptyMap = -``` - -## Decode properties -Given a sample.properties file of: -```properties -# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza - -``` -then -```bash -yq -p=props sample.properties -``` -will output -```yaml -person: - # block comments come through - # comments on values appear - name: Mike Wazowski - pets: - # comments on array values appear - - cat - food: - - pizza -``` - -## Decode properties - array should be a map -If you have a numeric map key in your property files, use array_to_map to convert them to maps. - -Given a sample.properties file of: -```properties -things.10 = mike -``` -then -```bash -yq -p=props '.things |= array_to_map' sample.properties -``` -will output -```yaml -things: - 10: mike -``` - -## Roundtrip -Given a sample.properties file of: -```properties -# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza - -``` -then -```bash -yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties -``` -will output -```properties -# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = dog -person.food.0 = pizza -``` - diff --git a/external/yq/pkg/yqlib/doc/usage/xml.md b/external/yq/pkg/yqlib/doc/usage/xml.md deleted file mode 100755 index 09f77b0..0000000 --- a/external/yq/pkg/yqlib/doc/usage/xml.md +++ /dev/null @@ -1,509 +0,0 @@ -# XML - -Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are. - -Consecutive xml nodes with the same name are assumed to be arrays. - -XML content data, attributes processing instructions and directives are all created as plain fields. - -This can be controlled by: - -| Flag | Default |Sample XML | -| -- | -- | -- | - | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | - | `--xml-content-name` | `+content` | Meow in ```Meow true``` | - | `--xml-directive-name` | `+directive` | `````` | - | `--xml-proc-inst-prefix` | `+p_` | `````` | - - -{% hint style="warning" %} -Default Attribute Prefix will be changing in v4.30! -In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@". - -This will affect users that have not set their own prefix and are not roundtripping XML changes. - -{% endhint %} - -## Encoder / Decoder flag options - -In addition to the above flags, there are the following xml encoder/decoder options controlled by flags: - -| Flag | Default | Description | -| -- | -- | -- | -| `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| -| `--xml-keep-namespace` | true | Keeps the namespace of attributes | -| `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. | -| `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` | -| `--xml-skip-directives` | false | Skips over directives, e.g. `````` | - - -See below for examples - -## Parse xml: simple -Notice how all the values are strings, see the next example on how you can fix that. - -Given a sample.xml file of: -```xml - - - meow - 4 - true - -``` -then -```bash -yq -p=xml '.' sample.xml -``` -will output -```yaml -+p_xml: version="1.0" encoding="UTF-8" -cat: - says: meow - legs: "4" - cute: "true" -``` - -## Parse xml: number -All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type. - -Given a sample.xml file of: -```xml - - - meow - 4 - true - -``` -then -```bash -yq -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml -``` -will output -```yaml -+p_xml: version="1.0" encoding="UTF-8" -cat: - says: meow - legs: 4 - cute: true -``` - -## Parse xml: array -Consecutive nodes with identical xml names are assumed to be arrays. - -Given a sample.xml file of: -```xml - -cat -goat -``` -then -```bash -yq -p=xml '.' sample.xml -``` -will output -```yaml -+p_xml: version="1.0" encoding="UTF-8" -animal: - - cat - - goat -``` - -## Parse xml: attributes -Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own. - -Given a sample.xml file of: -```xml - - - 7 - -``` -then -```bash -yq -p=xml '.' sample.xml -``` -will output -```yaml -+p_xml: version="1.0" encoding="UTF-8" -cat: - +@legs: "4" - legs: "7" -``` - -## Parse xml: attributes with content -Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own. - -Given a sample.xml file of: -```xml - -meow -``` -then -```bash -yq -p=xml '.' sample.xml -``` -will output -```yaml -+p_xml: version="1.0" encoding="UTF-8" -cat: - +content: meow - +@legs: "4" -``` - -## Parse xml: custom dtd -DTD entities are processed as directives. - -Given a sample.xml file of: -```xml - - - - -]> - - &writer;©right; - -``` -then -```bash -yq -p=xml -o=xml '.' sample.xml -``` -will output -```xml - - - -]> - - &writer;&copyright; - -``` - -## Parse xml: skip custom dtd -DTDs are directives, skip over directives to skip DTDs. - -Given a sample.xml file of: -```xml - - - - -]> - - &writer;©right; - -``` -then -```bash -yq -p=xml -o=xml --xml-skip-directives '.' sample.xml -``` -will output -```xml - - - &writer;&copyright; - -``` - -## Parse xml: with comments -A best attempt is made to preserve comments. - -Given a sample.xml file of: -```xml - - - - - 3 - - - - z - - - - - - - -``` -then -```bash -yq -p=xml '.' sample.xml -``` -will output -```yaml -# before cat -cat: - # in cat before - x: "3" # multi - # line comment - # for x - # before y - - y: - # in y before - # in d before - d: z # in d after - # in y after - # in_cat_after -# after cat -``` - -## Parse xml: keep attribute namespace -Defaults to true - -Given a sample.xml file of: -```xml - - - -``` -then -```bash -yq -p=xml -o=xml --xml-keep-namespace=false '.' sample.xml -``` -will output -```xml - - -``` - -instead of -```xml - - -``` - -## Parse xml: keep raw attribute namespace -Defaults to true - -Given a sample.xml file of: -```xml - - - -``` -then -```bash -yq -p=xml -o=xml --xml-raw-token=false '.' sample.xml -``` -will output -```xml - - -``` - -instead of -```xml - - -``` - -## Encode xml: simple -Given a sample.yml file of: -```yaml -cat: purrs -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml -purrs -``` - -## Encode xml: array -Given a sample.yml file of: -```yaml -pets: - cat: - - purrs - - meows -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml - - purrs - meows - -``` - -## Encode xml: attributes -Fields with the matching xml-attribute-prefix are assumed to be attributes. - -Given a sample.yml file of: -```yaml -cat: - +@name: tiger - meows: true - -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml - - true - -``` - -## Encode xml: attributes with content -Fields with the matching xml-content-name is assumed to be content. - -Given a sample.yml file of: -```yaml -cat: - +@name: tiger - +content: cool - -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml -cool -``` - -## Encode xml: comments -A best attempt is made to copy comments to xml. - -Given a sample.yml file of: -```yaml -# header comment -# above_cat -cat: # inline_cat - # above_array - array: # inline_array - - val1 # inline_val1 - # above_val2 - - val2 # inline_val2 -# below_cat - -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml - - val1 - val2 - -``` - -## Encode: doctype and xml declaration -Use the special xml names to add/modify proc instructions and directives. - -Given a sample.yml file of: -```yaml -+p_xml: version="1.0" -+directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" ' -apple: - +p_coolioo: version="1.0" - +directive: 'CATYPE meow purr puss ' - b: things - -``` -then -```bash -yq -o=xml '.' sample.yml -``` -will output -```xml - - - - things - -``` - -## Round trip: with comments -A best effort is made, but comment positions and white space are not preserved perfectly. - -Given a sample.xml file of: -```xml - - - - - 3 - - - - z - - - - - - - -``` -then -```bash -yq -p=xml -o=xml '.' sample.xml -``` -will output -```xml - - 3 - - z - - -``` - -## Roundtrip: with doctype and declaration -yq parses XML proc instructions and directives into nodes. -Unfortunately the underlying XML parser loses whitespace information. - -Given a sample.xml file of: -```xml - - - - - - things - - -``` -then -```bash -yq -p=xml -o=xml '.' sample.xml -``` -will output -```xml - - - - things - -``` - diff --git a/external/yq/pkg/yqlib/encoder.go b/external/yq/pkg/yqlib/encoder.go deleted file mode 100755 index e9fae6a..0000000 --- a/external/yq/pkg/yqlib/encoder.go +++ /dev/null @@ -1,178 +0,0 @@ -package yqlib - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - - yaml "gopkg.in/yaml.v3" -) - -type Encoder interface { - Encode(writer io.Writer, node *yaml.Node) error - PrintDocumentSeparator(writer io.Writer) error - PrintLeadingContent(writer io.Writer, content string) error - CanHandleAliases() bool -} - -// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the -// order of keys and values in a map or an object. -type orderedMap struct { - // if this is an object, kv != nil. If this is not an object, kv == nil. - kv []orderedMapKV - altVal interface{} -} - -type orderedMapKV struct { - K string - V orderedMap -} - -func (o *orderedMap) UnmarshalJSON(data []byte) error { - switch data[0] { - case '{': - // initialise so that even if the object is empty it is not nil - o.kv = []orderedMapKV{} - - // create decoder - dec := json.NewDecoder(bytes.NewReader(data)) - _, err := dec.Token() // open object - if err != nil { - return err - } - - // cycle through k/v - var tok json.Token - for tok, err = dec.Token(); err == nil; tok, err = dec.Token() { - // we can expect two types: string or Delim. Delim automatically means - // that it is the closing bracket of the object, whereas string means - // that there is another key. - if _, ok := tok.(json.Delim); ok { - break - } - kv := orderedMapKV{ - K: tok.(string), - } - if err := dec.Decode(&kv.V); err != nil { - return err - } - o.kv = append(o.kv, kv) - } - // unexpected error - if err != nil && !errors.Is(err, io.EOF) { - return err - } - return nil - case '[': - var res []*orderedMap - if err := json.Unmarshal(data, &res); err != nil { - return err - } - o.altVal = res - o.kv = nil - return nil - } - - return json.Unmarshal(data, &o.altVal) -} - -func (o orderedMap) MarshalJSON() ([]byte, error) { - buf := new(bytes.Buffer) - enc := json.NewEncoder(buf) - enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > - if o.kv == nil { - if err := enc.Encode(o.altVal); err != nil { - return nil, err - } - return buf.Bytes(), nil - } - buf.WriteByte('{') - for idx, el := range o.kv { - if err := enc.Encode(el.K); err != nil { - return nil, err - } - buf.WriteByte(':') - if err := enc.Encode(el.V); err != nil { - return nil, err - } - if idx != len(o.kv)-1 { - buf.WriteByte(',') - } - } - buf.WriteByte('}') - return buf.Bytes(), nil -} - -func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error { - switch node.Kind { - case yaml.DocumentNode: - if len(node.Content) == 0 { - return nil - } - return o.UnmarshalYAML(node.Content[0]) - case yaml.AliasNode: - return o.UnmarshalYAML(node.Alias) - case yaml.ScalarNode: - return node.Decode(&o.altVal) - case yaml.MappingNode: - // set kv to non-nil - o.kv = []orderedMapKV{} - for i := 0; i < len(node.Content); i += 2 { - var key string - var val orderedMap - if err := node.Content[i].Decode(&key); err != nil { - return err - } - if err := node.Content[i+1].Decode(&val); err != nil { - return err - } - o.kv = append(o.kv, orderedMapKV{ - K: key, - V: val, - }) - } - return nil - case yaml.SequenceNode: - // note that this has to be a pointer, so that nulls can be represented. - var res []*orderedMap - if err := node.Decode(&res); err != nil { - return err - } - o.altVal = res - o.kv = nil - return nil - case 0: - // null - o.kv = nil - o.altVal = nil - return nil - default: - return fmt.Errorf("orderedMap: invalid yaml node") - } -} - -func (o *orderedMap) MarshalYAML() (interface{}, error) { - // fast path: kv is nil, use altVal - if o.kv == nil { - return o.altVal, nil - } - content := make([]*yaml.Node, 0, len(o.kv)*2) - for _, val := range o.kv { - n := new(yaml.Node) - if err := n.Encode(val.V); err != nil { - return nil, err - } - content = append(content, &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: val.K, - }, n) - } - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: content, - }, nil -} diff --git a/external/yq/pkg/yqlib/encoder_base64.go b/external/yq/pkg/yqlib/encoder_base64.go deleted file mode 100755 index 635d27f..0000000 --- a/external/yq/pkg/yqlib/encoder_base64.go +++ /dev/null @@ -1,38 +0,0 @@ -package yqlib - -import ( - "encoding/base64" - "fmt" - "io" - - yaml "gopkg.in/yaml.v3" -) - -type base64Encoder struct { - encoding base64.Encoding -} - -func NewBase64Encoder() Encoder { - return &base64Encoder{encoding: *base64.StdEncoding} -} - -func (e *base64Encoder) CanHandleAliases() bool { - return false -} - -func (e *base64Encoder) PrintDocumentSeparator(writer io.Writer) error { - return nil -} - -func (e *base64Encoder) PrintLeadingContent(writer io.Writer, content string) error { - return nil -} - -func (e *base64Encoder) Encode(writer io.Writer, originalNode *yaml.Node) error { - node := unwrapDoc(originalNode) - if guessTagFromCustomType(node) != "!!str" { - return fmt.Errorf("cannot encode %v as base64, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) - } - _, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(originalNode.Value)))) - return err -} diff --git a/external/yq/pkg/yqlib/encoder_csv.go b/external/yq/pkg/yqlib/encoder_csv.go deleted file mode 100755 index 88e2b0d..0000000 --- a/external/yq/pkg/yqlib/encoder_csv.go +++ /dev/null @@ -1,126 +0,0 @@ -package yqlib - -import ( - "encoding/csv" - "fmt" - "io" - - yaml "gopkg.in/yaml.v3" -) - -type csvEncoder struct { - separator rune -} - -func NewCsvEncoder(separator rune) Encoder { - return &csvEncoder{separator: separator} -} - -func (e *csvEncoder) CanHandleAliases() bool { - return false -} - -func (e *csvEncoder) PrintDocumentSeparator(writer io.Writer) error { - return nil -} - -func (e *csvEncoder) PrintLeadingContent(writer io.Writer, content string) error { - return nil -} - -func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) error { - stringValues := make([]string, len(contents)) - - for i, child := range contents { - - if child.Kind != yaml.ScalarNode { - return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) - } - stringValues[i] = child.Value - } - return csvWriter.Write(stringValues) -} - -func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) error { - for i, child := range content { - - if child.Kind != yaml.SequenceNode { - return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) - } - err := e.encodeRow(csvWriter, child.Content) - if err != nil { - return err - } - } - return nil -} - -func (e *csvEncoder) extractHeader(child *yaml.Node) ([]*yaml.Node, error) { - if child.Kind != yaml.MappingNode { - return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag) - } - mapKeys := getMapKeys(child) - return mapKeys.Content, nil -} - -func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*yaml.Node { - childRow := make([]*yaml.Node, 0) - for _, header := range headers { - keyIndex := findKeyInMap(child, header) - value := createScalarNode(nil, "") - if keyIndex != -1 { - value = child.Content[keyIndex+1] - } - childRow = append(childRow, value) - } - return childRow - -} - -func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node) error { - headers, err := e.extractHeader(content[0]) - if err != nil { - return nil - } - - err = e.encodeRow(csvWriter, headers) - if err != nil { - return nil - } - - for i, child := range content { - if child.Kind != yaml.MappingNode { - return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag) - } - row := e.createChildRow(child, headers) - err = e.encodeRow(csvWriter, row) - if err != nil { - return err - } - - } - return nil -} - -func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error { - csvWriter := csv.NewWriter(writer) - csvWriter.Comma = e.separator - - // node must be a sequence - node := unwrapDoc(originalNode) - if node.Kind != yaml.SequenceNode { - return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag) - } else if len(node.Content) == 0 { - return nil - } - if node.Content[0].Kind == yaml.ScalarNode { - return e.encodeRow(csvWriter, node.Content) - } - - if node.Content[0].Kind == yaml.MappingNode { - return e.encodeObjects(csvWriter, node.Content) - } - - return e.encodeArrays(csvWriter, node.Content) - -} diff --git a/external/yq/pkg/yqlib/encoder_json.go b/external/yq/pkg/yqlib/encoder_json.go deleted file mode 100755 index 9f5b71a..0000000 --- a/external/yq/pkg/yqlib/encoder_json.go +++ /dev/null @@ -1,85 +0,0 @@ -package yqlib - -import ( - "bytes" - "io" - - "github.com/goccy/go-json" - yaml "gopkg.in/yaml.v3" -) - -type jsonEncoder struct { - indentString string - colorise bool - UnwrapScalar bool -} - -func mapKeysToStrings(node *yaml.Node) { - - if node.Kind == yaml.MappingNode { - for index, child := range node.Content { - if index%2 == 0 { // its a map key - child.Tag = "!!str" - } - } - } - - for _, child := range node.Content { - mapKeysToStrings(child) - } -} - -func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder { - var indentString = "" - - for index := 0; index < indent; index++ { - indentString = indentString + " " - } - - return &jsonEncoder{indentString, colorise, unwrapScalar} -} - -func (je *jsonEncoder) CanHandleAliases() bool { - return false -} - -func (je *jsonEncoder) PrintDocumentSeparator(writer io.Writer) error { - return nil -} - -func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) error { - return nil -} - -func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error { - - if node.Kind == yaml.ScalarNode && je.UnwrapScalar { - return writeString(writer, node.Value+"\n") - } - - destination := writer - tempBuffer := bytes.NewBuffer(nil) - if je.colorise { - destination = tempBuffer - } - - var encoder = json.NewEncoder(destination) - encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > - encoder.SetIndent("", je.indentString) - - var dataBucket orderedMap - // firstly, convert all map keys to strings - mapKeysToStrings(node) - errorDecoding := node.Decode(&dataBucket) - if errorDecoding != nil { - return errorDecoding - } - err := encoder.Encode(dataBucket) - if err != nil { - return err - } - if je.colorise { - return colorizeAndPrint(tempBuffer.Bytes(), writer) - } - return nil -} diff --git a/external/yq/pkg/yqlib/encoder_properties.go b/external/yq/pkg/yqlib/encoder_properties.go deleted file mode 100755 index e62b3f1..0000000 --- a/external/yq/pkg/yqlib/encoder_properties.go +++ /dev/null @@ -1,138 +0,0 @@ -package yqlib - -import ( - "bufio" - "errors" - "fmt" - "io" - "strings" - - "github.com/magiconair/properties" - yaml "gopkg.in/yaml.v3" -) - -type propertiesEncoder struct { - unwrapScalar bool -} - -func NewPropertiesEncoder(unwrapScalar bool) Encoder { - return &propertiesEncoder{ - unwrapScalar: unwrapScalar, - } -} - -func (pe *propertiesEncoder) CanHandleAliases() bool { - return false -} - -func (pe *propertiesEncoder) PrintDocumentSeparator(writer io.Writer) error { - return nil -} - -func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error { - reader := bufio.NewReader(strings.NewReader(content)) - for { - - readline, errReading := reader.ReadString('\n') - if errReading != nil && !errors.Is(errReading, io.EOF) { - return errReading - } - if strings.Contains(readline, "$yqDocSeperator$") { - - if err := pe.PrintDocumentSeparator(writer); err != nil { - return err - } - - } else { - if err := writeString(writer, readline); err != nil { - return err - } - } - - if errors.Is(errReading, io.EOF) { - if readline != "" { - // the last comment we read didn't have a new line, put one in - if err := writeString(writer, "\n"); err != nil { - return err - } - } - break - } - } - return nil -} - -func (pe *propertiesEncoder) Encode(writer io.Writer, node *yaml.Node) error { - mapKeysToStrings(node) - p := properties.NewProperties() - err := pe.doEncode(p, node, "", nil) - if err != nil { - return err - } - - _, err = p.WriteComment(writer, "#", properties.UTF8) - return err -} - -func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string, keyNode *yaml.Node) error { - - comments := "" - if keyNode != nil { - // include the key node comments if present - comments = headAndLineComment(keyNode) - } - comments = comments + headAndLineComment(node) - commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ") - p.SetComments(path, strings.Split(commentsWithSpaces, "\n")) - - switch node.Kind { - case yaml.ScalarNode: - var nodeValue string - if pe.unwrapScalar || !strings.Contains(node.Value, " ") { - nodeValue = node.Value - } else { - nodeValue = fmt.Sprintf("%q", node.Value) - } - _, _, err := p.Set(path, nodeValue) - return err - case yaml.DocumentNode: - return pe.doEncode(p, node.Content[0], path, node) - case yaml.SequenceNode: - return pe.encodeArray(p, node.Content, path) - case yaml.MappingNode: - return pe.encodeMap(p, node.Content, path) - case yaml.AliasNode: - return pe.doEncode(p, node.Alias, path, nil) - default: - return fmt.Errorf("Unsupported node %v", node.Tag) - } -} - -func (pe *propertiesEncoder) appendPath(path string, key interface{}) string { - if path == "" { - return fmt.Sprintf("%v", key) - } - return fmt.Sprintf("%v.%v", path, key) -} - -func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error { - for index, child := range kids { - err := pe.doEncode(p, child, pe.appendPath(path, index), nil) - if err != nil { - return err - } - } - return nil -} - -func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error { - for index := 0; index < len(kids); index = index + 2 { - key := kids[index] - value := kids[index+1] - err := pe.doEncode(p, value, pe.appendPath(path, key.Value), key) - if err != nil { - return err - } - } - return nil -} diff --git a/external/yq/pkg/yqlib/encoder_properties_test.go b/external/yq/pkg/yqlib/encoder_properties_test.go deleted file mode 100755 index d1ed5cf..0000000 --- a/external/yq/pkg/yqlib/encoder_properties_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "strings" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -func yamlToProps(sampleYaml string, unwrapScalar bool) string { - var output bytes.Buffer - writer := bufio.NewWriter(&output) - - var propsEncoder = NewPropertiesEncoder(unwrapScalar) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - node := inputs.Front().Value.(*CandidateNode).Node - err = propsEncoder.Encode(writer, node) - if err != nil { - panic(err) - } - writer.Flush() - - return strings.TrimSuffix(output.String(), "\n") -} - -func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool'` - - var expectedProps = `a = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderSimple_Wrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool'` - - var expectedProps = `a = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool' # line` - - var expectedProps = `# line -a = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) { - var sampleYaml = `a: 'bob cool' # line` - - var expectedProps = `# line -a = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) { - var sampleYaml = `a: - b: "bob cool" -` - - var expectedProps = `a.b = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeep_Wrapped(t *testing.T) { - var sampleYaml = `a: - b: "bob cool" -` - - var expectedProps = `a.b = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) { - var sampleYaml = `a: # a thing - b: "bob cool" # b thing -` - - var expectedProps = `# b thing -a.b = bob cool` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) { - var sampleYaml = `a: # a thing - b: "bob cool" # b thing -` - - var expectedProps = `# b thing -a.b = "bob cool"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderArray_Unwrapped(t *testing.T) { - var sampleYaml = `a: - b: [{c: dog}, {c: cat}] -` - - var expectedProps = `a.b.0.c = dog -a.b.1.c = cat` - var actualProps = yamlToProps(sampleYaml, true) - test.AssertResult(t, expectedProps, actualProps) -} - -func TestPropertiesEncoderArray_Wrapped(t *testing.T) { - var sampleYaml = `a: - b: [{c: dog named jim}, {c: cat named jam}] -` - - var expectedProps = `a.b.0.c = "dog named jim" -a.b.1.c = "cat named jam"` - var actualProps = yamlToProps(sampleYaml, false) - test.AssertResult(t, expectedProps, actualProps) -} diff --git a/external/yq/pkg/yqlib/encoder_test.go b/external/yq/pkg/yqlib/encoder_test.go deleted file mode 100755 index a17c51a..0000000 --- a/external/yq/pkg/yqlib/encoder_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "strings" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -func yamlToJSON(sampleYaml string, indent int) string { - var output bytes.Buffer - writer := bufio.NewWriter(&output) - - var jsonEncoder = NewJSONEncoder(indent, false, false) - inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - node := inputs.Front().Value.(*CandidateNode).Node - err = jsonEncoder.Encode(writer, node) - if err != nil { - panic(err) - } - writer.Flush() - - return strings.TrimSuffix(output.String(), "\n") -} - -func TestJSONEncoderPreservesObjectOrder(t *testing.T) { - var sampleYaml = `zabbix: winner -apple: great -banana: -- {cobra: kai, angus: bob} -` - var expectedJSON = `{ - "zabbix": "winner", - "apple": "great", - "banana": [ - { - "cobra": "kai", - "angus": "bob" - } - ] -}` - var actualJSON = yamlToJSON(sampleYaml, 2) - test.AssertResult(t, expectedJSON, actualJSON) -} - -func TestJsonNullInArray(t *testing.T) { - var sampleYaml = `[null]` - var actualJSON = yamlToJSON(sampleYaml, 0) - test.AssertResult(t, sampleYaml, actualJSON) -} - -func TestJsonNull(t *testing.T) { - var sampleYaml = `null` - var actualJSON = yamlToJSON(sampleYaml, 0) - test.AssertResult(t, sampleYaml, actualJSON) -} - -func TestJsonNullInObject(t *testing.T) { - var sampleYaml = `{x: null}` - var actualJSON = yamlToJSON(sampleYaml, 0) - test.AssertResult(t, `{"x":null}`, actualJSON) -} - -func TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) { - var sampleYaml = `build: "( ./lint && ./format && ./compile ) < src.code"` - var expectedJSON = `{"build":"( ./lint && ./format && ./compile ) < src.code"}` - var actualJSON = yamlToJSON(sampleYaml, 0) - test.AssertResult(t, expectedJSON, actualJSON) -} diff --git a/external/yq/pkg/yqlib/encoder_xml.go b/external/yq/pkg/yqlib/encoder_xml.go deleted file mode 100755 index 1d2e779..0000000 --- a/external/yq/pkg/yqlib/encoder_xml.go +++ /dev/null @@ -1,317 +0,0 @@ -package yqlib - -import ( - "encoding/xml" - "fmt" - "io" - "regexp" - "strings" - - yaml "gopkg.in/yaml.v3" -) - -type xmlEncoder struct { - indentString string - writer io.Writer - prefs XmlPreferences - leadingContent string -} - -var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`) - -func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder { - var indentString = "" - - for index := 0; index < indent; index++ { - indentString = indentString + " " - } - return &xmlEncoder{indentString, nil, prefs, ""} -} - -func (e *xmlEncoder) CanHandleAliases() bool { - return false -} - -func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error { - return nil -} - -func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error { - e.leadingContent = commentPrefix.ReplaceAllString(content, "\n") - return nil -} - -func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error { - encoder := xml.NewEncoder(writer) - // hack so we can manually add newlines to procInst and directives - e.writer = writer - encoder.Indent("", e.indentString) - - if e.leadingContent != "" { - err := e.encodeComment(encoder, e.leadingContent) - if err != nil { - return err - } - } - - switch node.Kind { - case yaml.MappingNode: - err := e.encodeTopLevelMap(encoder, node) - if err != nil { - return err - } - case yaml.DocumentNode: - err := e.encodeComment(encoder, headAndLineComment(node)) - if err != nil { - return err - } - unwrappedNode := unwrapDoc(node) - if unwrappedNode.Kind != yaml.MappingNode { - return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", unwrappedNode.Tag) - } - err = e.encodeTopLevelMap(encoder, unwrappedNode) - if err != nil { - return err - } - err = e.encodeComment(encoder, footComment(node)) - if err != nil { - return err - } - case yaml.ScalarNode: - var charData xml.CharData = []byte(node.Value) - err := encoder.EncodeToken(charData) - if err != nil { - return err - } - return encoder.Flush() - default: - return fmt.Errorf("unsupported type %v", node.Tag) - } - var charData xml.CharData = []byte("\n") - return encoder.EncodeToken(charData) - -} - -func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error { - // make sure processing instructions are encoded first - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - value := node.Content[i+1] - - if key.Value == (e.prefs.ProcInstPrefix + "xml") { - name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) - procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} - if err := encoder.EncodeToken(procInst); err != nil { - return err - } - if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) - } - } - } - - err := e.encodeComment(encoder, headAndLineComment(node)) - if err != nil { - return err - } - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - value := node.Content[i+1] - - start := xml.StartElement{Name: xml.Name{Local: key.Value}} - log.Debugf("comments of key %v", key.Value) - err := e.encodeComment(encoder, headAndLineComment(key)) - if err != nil { - return err - } - - if key.Value == (e.prefs.ProcInstPrefix + "xml") { - // dont double process these. - } else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) { - name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) - procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} - if err := encoder.EncodeToken(procInst); err != nil { - return err - } - if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) - } - } else if key.Value == e.prefs.DirectiveName { - var directive xml.Directive = []byte(value.Value) - if err := encoder.EncodeToken(directive); err != nil { - return err - } - if _, err := e.writer.Write([]byte("\n")); err != nil { - log.Warning("Unable to write newline, skipping: %w", err) - } - } else { - - log.Debugf("recursing") - - err = e.doEncode(encoder, value, start) - if err != nil { - return err - } - } - err = e.encodeComment(encoder, footComment(key)) - if err != nil { - return err - } - } - return e.encodeComment(encoder, footComment(node)) -} - -func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { - err := encoder.EncodeToken(start) - if err != nil { - return err - } - return e.encodeComment(encoder, headComment(node)) -} - -func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { - err := encoder.EncodeToken(start.End()) - if err != nil { - return err - } - return e.encodeComment(encoder, footComment(node)) -} - -func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { - switch node.Kind { - case yaml.MappingNode: - return e.encodeMap(encoder, node, start) - case yaml.SequenceNode: - return e.encodeArray(encoder, node, start) - case yaml.ScalarNode: - err := e.encodeStart(encoder, node, start) - if err != nil { - return err - } - - var charData xml.CharData = []byte(node.Value) - err = encoder.EncodeToken(charData) - if err != nil { - return err - } - - if err = e.encodeComment(encoder, lineComment(node)); err != nil { - return err - } - - return e.encodeEnd(encoder, node, start) - } - return fmt.Errorf("unsupported type %v", node.Tag) -} - -func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error { - if commentStr != "" { - log.Debugf("encoding comment %v", commentStr) - if !strings.HasSuffix(commentStr, " ") { - commentStr = commentStr + " " - } - - var comment xml.Comment = []byte(commentStr) - err := encoder.EncodeToken(comment) - if err != nil { - return err - } - } - return nil -} - -func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { - - if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil { - return err - } - - for i := 0; i < len(node.Content); i++ { - value := node.Content[i] - if err := e.doEncode(encoder, value, start.Copy()); err != nil { - return err - } - } - return e.encodeComment(encoder, footComment(node)) -} - -func (e *xmlEncoder) isAttribute(name string) bool { - return strings.HasPrefix(name, e.prefs.AttributePrefix) && - name != e.prefs.ContentName && - name != e.prefs.DirectiveName && - !strings.HasPrefix(name, e.prefs.ProcInstPrefix) -} - -func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yaml.Node, start xml.StartElement) error { - log.Debug("its a map") - - //first find all the attributes and put them on the start token - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - value := node.Content[i+1] - - if e.isAttribute(key.Value) { - if value.Kind == yaml.ScalarNode { - attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1) - start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value}) - } else { - return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag) - } - } - } - - err := e.encodeStart(encoder, node, start) - if err != nil { - return err - } - - //now we encode non attribute tokens - for i := 0; i < len(node.Content); i += 2 { - key := node.Content[i] - value := node.Content[i+1] - - err := e.encodeComment(encoder, headAndLineComment(key)) - if err != nil { - return err - } - if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) { - name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) - procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} - if err := encoder.EncodeToken(procInst); err != nil { - return err - } - } else if key.Value == e.prefs.DirectiveName { - var directive xml.Directive = []byte(value.Value) - if err := encoder.EncodeToken(directive); err != nil { - return err - } - } else if key.Value == e.prefs.ContentName { - // directly encode the contents - err = e.encodeComment(encoder, headAndLineComment(value)) - if err != nil { - return err - } - var charData xml.CharData = []byte(value.Value) - err = encoder.EncodeToken(charData) - if err != nil { - return err - } - err = e.encodeComment(encoder, footComment(value)) - if err != nil { - return err - } - } else if !e.isAttribute(key.Value) { - start := xml.StartElement{Name: xml.Name{Local: key.Value}} - err := e.doEncode(encoder, value, start) - if err != nil { - return err - } - } - err = e.encodeComment(encoder, footComment(key)) - if err != nil { - return err - } - } - - return e.encodeEnd(encoder, node, start) -} diff --git a/external/yq/pkg/yqlib/encoder_yaml.go b/external/yq/pkg/yqlib/encoder_yaml.go deleted file mode 100755 index ebf7ece..0000000 --- a/external/yq/pkg/yqlib/encoder_yaml.go +++ /dev/null @@ -1,100 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "errors" - "io" - "strings" - - yaml "gopkg.in/yaml.v3" -) - -type yamlEncoder struct { - indent int - colorise bool - prefs YamlPreferences -} - -func NewYamlEncoder(indent int, colorise bool, prefs YamlPreferences) Encoder { - if indent < 0 { - indent = 0 - } - return &yamlEncoder{indent, colorise, prefs} -} - -func (ye *yamlEncoder) CanHandleAliases() bool { - return true -} - -func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { - if ye.prefs.PrintDocSeparators { - log.Debug("-- writing doc sep") - if err := writeString(writer, "---\n"); err != nil { - return err - } - } - return nil -} - -func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { - // log.Debug("headcommentwas [%v]", content) - reader := bufio.NewReader(strings.NewReader(content)) - - for { - - readline, errReading := reader.ReadString('\n') - if errReading != nil && !errors.Is(errReading, io.EOF) { - return errReading - } - if strings.Contains(readline, "$yqDocSeperator$") { - - if err := ye.PrintDocumentSeparator(writer); err != nil { - return err - } - - } else { - if err := writeString(writer, readline); err != nil { - return err - } - } - - if errors.Is(errReading, io.EOF) { - if readline != "" { - // the last comment we read didn't have a new line, put one in - if err := writeString(writer, "\n"); err != nil { - return err - } - } - break - } - } - - return nil -} - -func (ye *yamlEncoder) Encode(writer io.Writer, node *yaml.Node) error { - - if node.Kind == yaml.ScalarNode && ye.prefs.UnwrapScalar { - return writeString(writer, node.Value+"\n") - } - - destination := writer - tempBuffer := bytes.NewBuffer(nil) - if ye.colorise { - destination = tempBuffer - } - - var encoder = yaml.NewEncoder(destination) - - encoder.SetIndent(ye.indent) - - if err := encoder.Encode(node); err != nil { - return err - } - - if ye.colorise { - return colorizeAndPrint(tempBuffer.Bytes(), writer) - } - return nil -} diff --git a/external/yq/pkg/yqlib/expression_parser.go b/external/yq/pkg/yqlib/expression_parser.go deleted file mode 100755 index 4722629..0000000 --- a/external/yq/pkg/yqlib/expression_parser.go +++ /dev/null @@ -1,76 +0,0 @@ -package yqlib - -import ( - "fmt" - "strings" -) - -type ExpressionNode struct { - Operation *Operation - LHS *ExpressionNode - RHS *ExpressionNode -} - -type ExpressionParserInterface interface { - ParseExpression(expression string) (*ExpressionNode, error) -} - -type expressionParserImpl struct { - pathTokeniser expressionTokeniser - pathPostFixer expressionPostFixer -} - -func newExpressionParser() ExpressionParserInterface { - return &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()} -} - -func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) { - log.Debug("Parsing expression: [%v]", expression) - tokens, err := p.pathTokeniser.Tokenise(expression) - if err != nil { - return nil, err - } - var Operations []*Operation - Operations, err = p.pathPostFixer.ConvertToPostfix(tokens) - if err != nil { - return nil, err - } - return p.createExpressionTree(Operations) -} - -func (p *expressionParserImpl) createExpressionTree(postFixPath []*Operation) (*ExpressionNode, error) { - var stack = make([]*ExpressionNode, 0) - - if len(postFixPath) == 0 { - return nil, nil - } - - for _, Operation := range postFixPath { - var newNode = ExpressionNode{Operation: Operation} - log.Debugf("pathTree %v ", Operation.toString()) - if Operation.OperationType.NumArgs > 0 { - numArgs := Operation.OperationType.NumArgs - if numArgs == 1 { - if len(stack) < 1 { - return nil, fmt.Errorf("'%v' expects 1 arg but received none", strings.TrimSpace(Operation.StringValue)) - } - remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] - newNode.RHS = rhs - stack = remaining - } else if numArgs == 2 { - if len(stack) < 2 { - return nil, fmt.Errorf("'%v' expects 2 args but there is %v", strings.TrimSpace(Operation.StringValue), len(stack)) - } - remaining, lhs, rhs := stack[:len(stack)-2], stack[len(stack)-2], stack[len(stack)-1] - newNode.LHS = lhs - newNode.RHS = rhs - stack = remaining - } - } - stack = append(stack, &newNode) - } - if len(stack) != 1 { - return nil, fmt.Errorf("bad expression, please check expression syntax") - } - return stack[0], nil -} diff --git a/external/yq/pkg/yqlib/expression_parser_test.go b/external/yq/pkg/yqlib/expression_parser_test.go deleted file mode 100755 index 2718a16..0000000 --- a/external/yq/pkg/yqlib/expression_parser_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package yqlib - -import ( - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -func getExpressionParser() ExpressionParserInterface { - InitExpressionParser() - return ExpressionParser -} - -func TestParserNoMatchingCloseBracket(t *testing.T) { - _, err := getExpressionParser().ParseExpression(".cat | with(.;.bob") - test.AssertResultComplex(t, "bad expression - probably missing close bracket on WITH", err.Error()) -} - -func TestParserNoMatchingCloseCollect(t *testing.T) { - _, err := getExpressionParser().ParseExpression("[1,2") - test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error()) -} -func TestParserNoMatchingCloseObjectInCollect(t *testing.T) { - _, err := getExpressionParser().ParseExpression(`[{"b": "c"]`) - test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error()) -} - -func TestParserNoMatchingCloseInCollect(t *testing.T) { - _, err := getExpressionParser().ParseExpression(`[(.a]`) - test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error()) -} - -func TestParserNoMatchingCloseCollectObject(t *testing.T) { - _, err := getExpressionParser().ParseExpression(`{"a": "b"`) - test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error()) -} - -func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) { - _, err := getExpressionParser().ParseExpression(`{"b": [1}`) - test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error()) -} - -func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) { - _, err := getExpressionParser().ParseExpression(`{"b": (1}`) - test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error()) -} - -func TestParserNoArgsForTwoArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression("=") - test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error()) -} - -func TestParserOneLhsArgsForTwoArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression(".a =") - test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) -} - -func TestParserOneRhsArgsForTwoArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression("= .a") - test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) -} - -func TestParserTwoArgsForTwoArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression(".a = .b") - test.AssertResultComplex(t, nil, err) -} - -func TestParserNoArgsForOneArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression("explode") - test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error()) -} - -func TestParserOneArgForOneArgOp(t *testing.T) { - _, err := getExpressionParser().ParseExpression("explode(.)") - test.AssertResultComplex(t, nil, err) -} - -func TestParserExtraArgs(t *testing.T) { - _, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)") - test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error()) -} diff --git a/external/yq/pkg/yqlib/expression_postfix.go b/external/yq/pkg/yqlib/expression_postfix.go deleted file mode 100755 index fa7afa3..0000000 --- a/external/yq/pkg/yqlib/expression_postfix.go +++ /dev/null @@ -1,144 +0,0 @@ -package yqlib - -import ( - "errors" - "fmt" - - logging "gopkg.in/op/go-logging.v1" -) - -type expressionPostFixer interface { - ConvertToPostfix([]*token) ([]*Operation, error) -} - -type expressionPostFixerImpl struct { -} - -func newExpressionPostFixer() expressionPostFixer { - return &expressionPostFixerImpl{} -} - -func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operation) { - var newOp *token - opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1] - log.Debugf("popped %v from opstack to results", newOp.toString(true)) - return opStack, append(result, newOp.Operation) -} - -func validateNoOpenTokens(token *token) error { - if token.TokenType == openCollect { - return fmt.Errorf(("bad expression, could not find matching `]`")) - } else if token.TokenType == openCollectObject { - return fmt.Errorf(("bad expression, could not find matching `}`")) - } else if token.TokenType == openBracket { - return fmt.Errorf(("bad expression, could not find matching `)`")) - } - return nil -} - -func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) { - var result []*Operation - // surround the whole thing with brackets - var opStack = []*token{{TokenType: openBracket}} - var tokens = append(infixTokens, &token{TokenType: closeBracket}) - - for _, currentToken := range tokens { - log.Debugf("postfix processing currentToken %v", currentToken.toString(true)) - switch currentToken.TokenType { - case openBracket, openCollect, openCollectObject: - opStack = append(opStack, currentToken) - log.Debugf("put %v onto the opstack", currentToken.toString(true)) - case closeCollect, closeCollectObject: - var opener tokenType = openCollect - var collectOperator = collectOpType - if currentToken.TokenType == closeCollectObject { - opener = openCollectObject - collectOperator = collectObjectOpType - } - - for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener { - missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1]) - if missingClosingTokenErr != nil { - return nil, missingClosingTokenErr - } - opStack, result = popOpToResult(opStack, result) - } - if len(opStack) == 0 { - return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket") - } - // now we should have [ as the last element on the opStack, get rid of it - opStack = opStack[0 : len(opStack)-1] - log.Debugf("deleting open bracket from opstack") - - //and append a collect to the result - - // hack - see if there's the optional traverse flag - // on the close op - move it to the traverse array op - // allows for .["cat"]? - prefs := traversePreferences{} - closeTokenMatch := currentToken.Match - if closeTokenMatch[len(closeTokenMatch)-1:] == "?" { - prefs.OptionalTraverse = true - } - result = append(result, &Operation{OperationType: collectOperator}) - log.Debugf("put collect onto the result") - if opener != openCollect { - result = append(result, &Operation{OperationType: shortPipeOpType}) - log.Debugf("put shortpipe onto the result") - } - - //traverseArrayCollect is a sneaky op that needs to be included too - //when closing a ] - if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType { - opStack[len(opStack)-1].Operation.Preferences = prefs - opStack, result = popOpToResult(opStack, result) - } - - case closeBracket: - for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket { - missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1]) - if missingClosingTokenErr != nil { - return nil, missingClosingTokenErr - } - - opStack, result = popOpToResult(opStack, result) - } - if len(opStack) == 0 { - return nil, errors.New("bad expression, got close brackets without matching opening bracket") - } - // now we should have ( as the last element on the opStack, get rid of it - opStack = opStack[0 : len(opStack)-1] - - default: - var currentPrecedence = currentToken.Operation.OperationType.Precedence - // pop off higher precedent operators onto the result - for len(opStack) > 0 && - opStack[len(opStack)-1].TokenType == operationToken && - opStack[len(opStack)-1].Operation.OperationType.Precedence > currentPrecedence { - opStack, result = popOpToResult(opStack, result) - } - // add this operator to the opStack - opStack = append(opStack, currentToken) - log.Debugf("put %v onto the opstack", currentToken.toString(true)) - } - } - - log.Debugf("opstackLen: %v", len(opStack)) - if len(opStack) > 0 { - log.Debugf("opstack:") - for _, token := range opStack { - log.Debugf("- %v", token.toString(true)) - } - - return nil, fmt.Errorf("bad expression - probably missing close bracket on %v", opStack[len(opStack)-1].toString(false)) - } - - if log.IsEnabledFor(logging.DEBUG) { - log.Debugf("PostFix Result:") - for _, currentToken := range result { - log.Debugf("> %v", currentToken.toString()) - } - } - - return result, nil -} diff --git a/external/yq/pkg/yqlib/expression_processing_test.go b/external/yq/pkg/yqlib/expression_processing_test.go deleted file mode 100755 index ffc6700..0000000 --- a/external/yq/pkg/yqlib/expression_processing_test.go +++ /dev/null @@ -1,344 +0,0 @@ -package yqlib - -import ( - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -var variableWithNewLine = `"cat -"` - -var pathTests = []struct { - path string - expectedTokens []interface{} - expectedPostFix []interface{} -}{ - { - `envsubst(ne)`, - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), - }, - { - `envsubst(nu)`, - append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), - append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), - }, - { - `envsubst(nu, ne)`, - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), - }, - { - `envsubst(ne, nu)`, - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), - append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), - }, - { - `[.a, .b]`, - append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"), - append(make([]interface{}, 0), "a", "b", "UNION", "COLLECT"), - }, - { - `.[env(myenv)]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "ENV", "]"), - append(make([]interface{}, 0), "SELF", "ENV", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.["cat"].["dog"]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"), - append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), - }, - { - `.["cat"]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"), - append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - "with(.a;.=3)", - append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"), - append(make([]interface{}, 0), "a", "SELF", "3 (int64)", "ASSIGN", "BLOCK", "WITH"), - }, - { - "0x12", - append(make([]interface{}, 0), "18 (int64)"), - append(make([]interface{}, 0), "18 (int64)"), - }, - { - "0X12", - append(make([]interface{}, 0), "18 (int64)"), - append(make([]interface{}, 0), "18 (int64)"), - }, - { - ".a\n", - append(make([]interface{}, 0), "a"), - append(make([]interface{}, 0), "a"), - }, - { - variableWithNewLine, - append(make([]interface{}, 0), "cat\n (string)"), - append(make([]interface{}, 0), "cat\n (string)"), - }, - { - `.[0]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), - append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.[0][1]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"), - append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `"\""`, - append(make([]interface{}, 0), "\" (string)"), - append(make([]interface{}, 0), "\" (string)"), - }, - { - `[]|join(".")`, - append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"), - append(make([]interface{}, 0), "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE"), - }, - { - `{"cool": .b or .c}`, - append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"), - append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `{"cool": []|join(".")}`, - append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"), - append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix - append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"), - append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"), - }, - { - `.a | .b | .c`, - append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"), - append(make([]interface{}, 0), "a", "b", "c", "PIPE", "PIPE"), - }, - { - `[]`, - append(make([]interface{}, 0), "[", "EMPTY", "]"), - append(make([]interface{}, 0), "EMPTY", "COLLECT"), - }, - { - `{}`, - append(make([]interface{}, 0), "{", "EMPTY", "}"), - append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `[{}]`, - append(make([]interface{}, 0), "[", "{", "EMPTY", "}", "]"), - append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT"), - }, - { - `.realnames as $names | $names["anon"]`, - append(make([]interface{}, 0), "realnames", "ASSIGN_VARIABLE", "GET_VARIABLE", "PIPE", "GET_VARIABLE", "TRAVERSE_ARRAY", "[", "anon (string)", "]"), - append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "TRAVERSE_ARRAY", "PIPE"), - }, - { - `.b[.a]`, - append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"), - append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.b[.a]?`, - append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"), - append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.[]`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), - append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.a[]`, - append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), - append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.a[]?`, - append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), - append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.a.[]`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), - append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), - }, - { - `.a[0]`, - append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), - append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.a[0]?`, - append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), - append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), - }, - { - `.a.[0]`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), - append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), - }, - { - `.a[].c`, - append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "c"), - append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"), - }, - { - `[3]`, - append(make([]interface{}, 0), "[", "3 (int64)", "]"), - append(make([]interface{}, 0), "3 (int64)", "COLLECT"), - }, - { - `.key.array + .key.array2`, - append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ADD", "key", "SHORT_PIPE", "array2"), - append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ADD"), - }, - { - `.key.array * .key.array2`, - append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "MULTIPLY", "key", "SHORT_PIPE", "array2"), - append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "MULTIPLY"), - }, - { - `.key.array // .key.array2`, - append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ALTERNATIVE", "key", "SHORT_PIPE", "array2"), - append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ALTERNATIVE"), - }, - { - `.a | .[].b == "apple"`, - append(make([]interface{}, 0), "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"), - append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"), - }, - { - `(.a | .[].b) == "apple"`, - append(make([]interface{}, 0), "(", "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"), - append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"), - }, - { - `.[] | select(. == "*at")`, - append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"), - append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"), - }, - { - `[true]`, - append(make([]interface{}, 0), "[", "true (bool)", "]"), - append(make([]interface{}, 0), "true (bool)", "COLLECT"), - }, - { - `[true, false]`, - append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"), - append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT"), - }, - { - `"mike": .a`, - append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"), - append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP"), - }, - { - `.a: "mike"`, - append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"), - append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"), - }, - { - `{"mike": .a}`, - append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"), - append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `{.a: "mike"}`, - append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"), - append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `{.a: .c, .b.[]: .f.g[]}`, - append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"), - append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"), - }, - { - `explode(.a.b)`, - append(make([]interface{}, 0), "EXPLODE", "(", "a", "SHORT_PIPE", "b", ")"), - append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "EXPLODE"), - }, - { - `.a.b style="folded"`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_STYLE", "folded (string)"), - append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "folded (string)", "ASSIGN_STYLE"), - }, - { - `tag == "str"`, - append(make([]interface{}, 0), "GET_TAG", "EQUALS", "str (string)"), - append(make([]interface{}, 0), "GET_TAG", "str (string)", "EQUALS"), - }, - { - `. tag= "str"`, - append(make([]interface{}, 0), "SELF", "ASSIGN_TAG", "str (string)"), - append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_TAG"), - }, - { - `lineComment == "str"`, - append(make([]interface{}, 0), "GET_COMMENT", "EQUALS", "str (string)"), - append(make([]interface{}, 0), "GET_COMMENT", "str (string)", "EQUALS"), - }, - { - `. lineComment= "str"`, - append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"), - append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"), - }, - { - `. lineComment |= "str"`, - append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"), - append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"), - }, - { - `.a.b tag="!!str"`, - append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"), - append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "!!str (string)", "ASSIGN_TAG"), - }, - { - `""`, - append(make([]interface{}, 0), " (string)"), - append(make([]interface{}, 0), " (string)"), - }, - { - `.foo* | (. style="flow")`, - append(make([]interface{}, 0), "foo*", "PIPE", "(", "SELF", "ASSIGN_STYLE", "flow (string)", ")"), - append(make([]interface{}, 0), "foo*", "SELF", "flow (string)", "ASSIGN_STYLE", "PIPE"), - }, -} - -var tokeniser = newParticipleLexer() -var postFixer = newExpressionPostFixer() - -func TestPathParsing(t *testing.T) { - for _, tt := range pathTests { - tokens, err := tokeniser.Tokenise(tt.path) - if err != nil { - t.Error(tt.path, err) - } - var tokenValues []interface{} - for _, token := range tokens { - tokenValues = append(tokenValues, token.toString(false)) - } - test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path)) - - results, errorP := postFixer.ConvertToPostfix(tokens) - - var readableResults []interface{} - for _, token := range results { - readableResults = append(readableResults, token.toString()) - } - - if errorP != nil { - t.Error(tt.path, err) - } - - test.AssertResultComplexWithContext(t, tt.expectedPostFix, readableResults, fmt.Sprintf("postfix: %v", tt.path)) - - } -} diff --git a/external/yq/pkg/yqlib/file_utils.go b/external/yq/pkg/yqlib/file_utils.go deleted file mode 100755 index 307e459..0000000 --- a/external/yq/pkg/yqlib/file_utils.go +++ /dev/null @@ -1,85 +0,0 @@ -package yqlib - -import ( - "fmt" - "io" - "os" -) - -func tryRenameFile(from string, to string) error { - if renameError := os.Rename(from, to); renameError != nil { - log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) - log.Debug(renameError.Error()) - log.Debug("going to try copying instead") - // can't do this rename when running in docker to a file targeted in a mounted volume, - // so gracefully degrade to copying the entire contents. - if copyError := copyFileContents(from, to); copyError != nil { - return fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError) - } - tryRemoveTempFile(from) - } - return nil -} - -func tryRemoveTempFile(filename string) { - log.Debug("Removing temp file: %v", filename) - removeErr := os.Remove(filename) - if removeErr != nil { - log.Errorf("Failed to remove temp file: %v", filename) - } -} - -// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang -func copyFileContents(src, dst string) (err error) { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside thar directory. - - in, err := os.Open(src) // #nosec - if err != nil { - return err - } - defer safelyCloseFile(in) - out, err := os.Create(dst) // #nosec - if err != nil { - return err - } - defer safelyCloseFile(out) - if _, err = io.Copy(out, in); err != nil { - return err - } - return out.Sync() -} - -func SafelyCloseReader(reader io.Reader) { - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } -} - -func safelyCloseFile(file *os.File) { - err := file.Close() - if err != nil { - log.Error("Error closing file!") - log.Error(err.Error()) - } -} - -func createTempFile() (*os.File, error) { - _, err := os.Stat(os.TempDir()) - if os.IsNotExist(err) { - err = os.Mkdir(os.TempDir(), 0700) - if err != nil { - return nil, err - } - } else if err != nil { - return nil, err - } - - file, err := os.CreateTemp("", "temp") - if err != nil { - return nil, err - } - - return file, err -} diff --git a/external/yq/pkg/yqlib/front_matter.go b/external/yq/pkg/yqlib/front_matter.go deleted file mode 100755 index e192f44..0000000 --- a/external/yq/pkg/yqlib/front_matter.go +++ /dev/null @@ -1,94 +0,0 @@ -package yqlib - -import ( - "bufio" - "errors" - "io" - "os" -) - -type frontMatterHandler interface { - Split() error - GetYamlFrontMatterFilename() string - GetContentReader() io.Reader - CleanUp() -} - -type frontMatterHandlerImpl struct { - originalFilename string - yamlFrontMatterFilename string - contentReader io.Reader -} - -func NewFrontMatterHandler(originalFilename string) frontMatterHandler { - return &frontMatterHandlerImpl{originalFilename, "", nil} -} - -func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string { - return f.yamlFrontMatterFilename -} - -func (f *frontMatterHandlerImpl) GetContentReader() io.Reader { - return f.contentReader -} - -func (f *frontMatterHandlerImpl) CleanUp() { - tryRemoveTempFile(f.yamlFrontMatterFilename) -} - -// Splits the given file by yaml front matter -// yaml content will be saved to first temporary file -// remaining content will be saved to second temporary file -func (f *frontMatterHandlerImpl) Split() error { - var reader *bufio.Reader - var err error - if f.originalFilename == "-" { - reader = bufio.NewReader(os.Stdin) - } else { - file, err := os.Open(f.originalFilename) // #nosec - if err != nil { - return err - } - reader = bufio.NewReader(file) - } - f.contentReader = reader - - yamlTempFile, err := createTempFile() - if err != nil { - return err - } - f.yamlFrontMatterFilename = yamlTempFile.Name() - log.Debug("yamlTempFile: %v", yamlTempFile.Name()) - - lineCount := 0 - - for { - peekBytes, err := reader.Peek(3) - if errors.Is(err, io.EOF) { - // we've finished reading the yaml content..I guess - break - } else if err != nil { - return err - } - if lineCount > 0 && string(peekBytes) == "---" { - // we've finished reading the yaml content.. - break - } - line, errReading := reader.ReadString('\n') - lineCount = lineCount + 1 - if errReading != nil && !errors.Is(errReading, io.EOF) { - return errReading - } - - _, errWriting := yamlTempFile.WriteString(line) - - if errWriting != nil { - return errWriting - } - } - - safelyCloseFile(yamlTempFile) - - return nil - -} diff --git a/external/yq/pkg/yqlib/front_matter_test.go b/external/yq/pkg/yqlib/front_matter_test.go deleted file mode 100755 index 47547c5..0000000 --- a/external/yq/pkg/yqlib/front_matter_test.go +++ /dev/null @@ -1,143 +0,0 @@ -package yqlib - -import ( - "io" - "os" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -func createTestFile(content string) string { - tempFile, err := createTempFile() - if err != nil { - panic(err) - } - - _, err = tempFile.Write([]byte(content)) - if err != nil { - panic(err) - } - - safelyCloseFile(tempFile) - - return tempFile.Name() -} - -func readFile(filename string) string { - bytes, err := os.ReadFile(filename) - if err != nil { - panic(err) - } - return string(bytes) -} - -func TestFrontMatterSplitWithLeadingSep(t *testing.T) { - file := createTestFile(`--- -a: apple -b: banana ---- -not a -yaml: doc -`) - - expectedYamlFm := `--- -a: apple -b: banana -` - - expectedContent := `--- -not a -yaml: doc -` - - fmHandler := NewFrontMatterHandler(file) - err := fmHandler.Split() - if err != nil { - panic(err) - } - - yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) - - test.AssertResult(t, expectedYamlFm, yamlFm) - - contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) - if err != nil { - panic(err) - } - test.AssertResult(t, expectedContent, string(contentBytes)) - - tryRemoveTempFile(file) - fmHandler.CleanUp() -} - -func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) { - file := createTestFile(`a: apple -b: banana ---- -not a -yaml: doc -`) - - expectedYamlFm := `a: apple -b: banana -` - - expectedContent := `--- -not a -yaml: doc -` - - fmHandler := NewFrontMatterHandler(file) - err := fmHandler.Split() - if err != nil { - panic(err) - } - - yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) - - test.AssertResult(t, expectedYamlFm, yamlFm) - - contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) - if err != nil { - panic(err) - } - test.AssertResult(t, expectedContent, string(contentBytes)) - - tryRemoveTempFile(file) - fmHandler.CleanUp() -} - -func TestFrontMatterSplitWithArray(t *testing.T) { - file := createTestFile(`[1,2,3] ---- -not a -yaml: doc -`) - - expectedYamlFm := "[1,2,3]\n" - - expectedContent := `--- -not a -yaml: doc -` - - fmHandler := NewFrontMatterHandler(file) - err := fmHandler.Split() - if err != nil { - panic(err) - } - - yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) - - test.AssertResult(t, expectedYamlFm, yamlFm) - - contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) - if err != nil { - panic(err) - } - test.AssertResult(t, expectedContent, string(contentBytes)) - - tryRemoveTempFile(file) - fmHandler.CleanUp() -} diff --git a/external/yq/pkg/yqlib/json_test.go b/external/yq/pkg/yqlib/json_test.go deleted file mode 100755 index 66ec114..0000000 --- a/external/yq/pkg/yqlib/json_test.go +++ /dev/null @@ -1,413 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three -b: - c: 2 - d: - - 3 - - 4 -` - -const sampleNdJson = `{"this": "is a multidoc json file"} -{"each": ["line is a valid json document"]} -{"a number": 4} -` - -const sampleNdJsonKey = `{"a": "first", "b": "next", "ab": "last"}` - -const expectedJsonKeysInOrder = `a: first -b: next -ab: last -` - -const expectedNdJsonYaml = `this: is a multidoc json file ---- -each: - - line is a valid json document ---- -a number: 4 -` - -const expectedRoundTripSampleNdJson = `{"this":"is a multidoc json file"} -{"each":["line is a valid json document"]} -{"a number":4} -` - -const expectedUpdatedMultilineJson = `{"this":"is a multidoc json file"} -{"each":["line is a valid json document","cool"]} -{"a number":4} -` - -const sampleMultiLineJson = `{ - "this": "is a multidoc json file" -} -{ - "it": [ - "has", - "consecutive", - "json documents" - ] -} -{ - "a number": 4 -} -` - -const roundTripMultiLineJson = `{ - "this": "is a multidoc json file" -} -{ - "it": [ - "has", - "consecutive", - "json documents" - ] -} -{ - "a number": 4 -} -` - -var jsonScenarios = []formatScenario{ - { - description: "Parse json: simple", - subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", - input: `{"cat": "meow"}`, - expected: "D0, P[], (!!map)::cat: meow\n", - }, - { - description: "bad json", - skipDoc: true, - input: `{"a": 1 "b": 2}`, - expectedError: `bad file 'sample.yml': invalid character '"' after object key:value pair`, - scenarioType: "decode-error", - }, - { - description: "Parse json: complex", - subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", - input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`, - expected: complexExpectYaml, - }, - { - description: "Encode json: simple", - input: `cat: meow`, - indent: 2, - expected: "{\n \"cat\": \"meow\"\n}\n", - scenarioType: "encode", - }, - { - description: "Encode json: simple - in one line", - input: `cat: meow # this is a comment, and it will be dropped.`, - indent: 0, - expected: "{\"cat\":\"meow\"}\n", - scenarioType: "encode", - }, - { - description: "Encode json: comments", - input: `cat: meow # this is a comment, and it will be dropped.`, - indent: 2, - expected: "{\n \"cat\": \"meow\"\n}\n", - scenarioType: "encode", - }, - { - description: "Encode json: anchors", - subdescription: "Anchors are dereferenced", - input: "cat: &ref meow\nanotherCat: *ref", - indent: 2, - expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n", - scenarioType: "encode", - }, - { - description: "Encode json: multiple results", - subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)", - input: `things: [{stuff: cool}, {whatever: cat}]`, - expression: `.things[]`, - indent: 0, - expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n", - scenarioType: "encode", - }, - { - description: "Roundtrip NDJSON", - subdescription: "Unfortunately the json encoder strips leading spaces of values.", - input: sampleNdJson, - expected: expectedRoundTripSampleNdJson, - scenarioType: "roundtrip-ndjson", - }, - { - description: "Roundtrip multi-document JSON", - subdescription: "The NDJSON parser can also handle multiple multi-line json documents in a single file!", - input: sampleMultiLineJson, - expected: roundTripMultiLineJson, - scenarioType: "roundtrip-multi", - }, - { - description: "Update a specific document in a multi-document json", - subdescription: "Documents are indexed by the `documentIndex` or `di` operator.", - input: sampleNdJson, - expected: expectedUpdatedMultilineJson, - expression: `(select(di == 1) | .each ) += "cool"`, - scenarioType: "roundtrip-ndjson", - }, - { - description: "Find and update a specific document in a multi-document json", - subdescription: "Use expressions as you normally would.", - input: sampleNdJson, - expected: expectedUpdatedMultilineJson, - expression: `(select(has("each")) | .each ) += "cool"`, - scenarioType: "roundtrip-ndjson", - }, - { - description: "Decode NDJSON", - input: sampleNdJson, - expected: expectedNdJsonYaml, - scenarioType: "decode-ndjson", - }, - { - description: "Decode NDJSON, maintain key order", - skipDoc: true, - input: sampleNdJsonKey, - expected: expectedJsonKeysInOrder, - scenarioType: "decode-ndjson", - }, - { - description: "numbers", - skipDoc: true, - input: "[3, 3.0, 3.1, -1]", - expected: "- 3\n- 3\n- 3.1\n- -1\n", - scenarioType: "decode-ndjson", - }, - { - description: "number single", - skipDoc: true, - input: "3", - expected: "3\n", - scenarioType: "decode-ndjson", - }, - { - description: "empty string", - skipDoc: true, - input: `""`, - expected: "\"\"\n", - scenarioType: "decode-ndjson", - }, - { - description: "strings", - skipDoc: true, - input: `["", "cat"]`, - expected: "- \"\"\n- cat\n", - scenarioType: "decode-ndjson", - }, - { - description: "null", - skipDoc: true, - input: `null`, - expected: "null\n", - scenarioType: "decode-ndjson", - }, - { - description: "booleans", - skipDoc: true, - input: `[true, false]`, - expected: "- true\n- false\n", - scenarioType: "decode-ndjson", - }, -} - -func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.json file of:\n") - writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v '%v' sample.json\n```\n", indent, expression)) - } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v sample.json\n```\n", indent)) - } - - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(indent, false, false)))) -} - -func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.json file of:\n") - writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json '%v' sample.json\n```\n", expression)) - } else { - writeOrPanic(w, "```bash\nyq -p=json sample.json\n```\n") - } - - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) -} - -func decodeJSON(t *testing.T, jsonString string) *CandidateNode { - docs, err := readDocument(jsonString, "sample.json", 0) - - if err != nil { - t.Error(err) - return nil - } - - exp, err := getExpressionParser().ParseExpression(PrettyPrintExp) - - if err != nil { - t.Error(err) - return nil - } - - context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: docs}, exp) - - if err != nil { - t.Error(err) - return nil - } - - return context.MatchingNodes.Front().Value.(*CandidateNode) -} - -func testJSONScenario(t *testing.T, s formatScenario) { - switch s.scenarioType { - case "encode", "decode": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)), s.description) - case "": - var actual = resultToString(t, decodeJSON(t, s.input)) - test.AssertResultWithContext(t, s.expected, actual, s.description) - case "decode-ndjson": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "roundtrip-ndjson": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(0, false, false)), s.description) - case "roundtrip-multi": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)), s.description) - case "decode-error": - result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(2, false, false)) - if err == nil { - t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) - } else { - test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) - } - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.json file of:\n") - writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -P '.' sample.json\n```\n") - writeOrPanic(w, "will output\n") - - var output bytes.Buffer - printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) - - node := decodeJSON(t, s.input) - - err := printer.PrintResults(node.AsList()) - if err != nil { - t.Error(err) - return - } - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String())) -} - -func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) { - s := i.(formatScenario) - if s.skipDoc { - return - } - switch s.scenarioType { - case "": - documentJSONDecodeScenario(t, w, s) - case "encode": - documentJSONEncodeScenario(w, s) - case "decode-ndjson": - documentDecodeNdJsonScenario(w, s) - case "roundtrip-ndjson": - documentRoundtripNdJsonScenario(w, s, 0) - case "roundtrip-multi": - documentRoundtripNdJsonScenario(w, s, 2) - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - if expression == "" { - expression = "." - } - - if s.indent == 2 { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json '%v' sample.yml\n```\n", expression)) - } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression)) - } - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(s.indent, false, false)))) -} - -func TestJSONScenarios(t *testing.T) { - for _, tt := range jsonScenarios { - testJSONScenario(t, tt) - } - genericScenarios := make([]interface{}, len(jsonScenarios)) - for i, s := range jsonScenarios { - genericScenarios[i] = s - } - documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario) -} diff --git a/external/yq/pkg/yqlib/lexer.go b/external/yq/pkg/yqlib/lexer.go deleted file mode 100755 index 259dbd3..0000000 --- a/external/yq/pkg/yqlib/lexer.go +++ /dev/null @@ -1,183 +0,0 @@ -package yqlib - -import ( - "fmt" - "regexp" -) - -type expressionTokeniser interface { - Tokenise(expression string) ([]*token, error) -} - -type tokenType uint32 - -const ( - operationToken = 1 << iota - openBracket - closeBracket - openCollect - closeCollect - openCollectObject - closeCollectObject - traverseArrayCollect -) - -type token struct { - TokenType tokenType - Operation *Operation - AssignOperation *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it - CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat - Match string -} - -func (t *token) toString(detail bool) string { - if t.TokenType == operationToken { - if detail { - return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence) - } - return t.Operation.toString() - } else if t.TokenType == openBracket { - return "(" - } else if t.TokenType == closeBracket { - return ")" - } else if t.TokenType == openCollect { - return "[" - } else if t.TokenType == closeCollect { - return "]" - } else if t.TokenType == openCollectObject { - return "{" - } else if t.TokenType == closeCollectObject { - return "}" - } else if t.TokenType == traverseArrayCollect { - return ".[" - - } else { - return "NFI" - } -} - -func unwrap(value string) string { - return value[1 : len(value)-1] -} - -func extractNumberParameter(value string) (int, error) { - parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`) - matches := parameterParser.FindStringSubmatch(value) - var indent, errParsingInt = parseInt(matches[1]) - if errParsingInt != nil { - return 0, errParsingInt - } - return indent, nil -} - -func hasOptionParameter(value string, option string) bool { - parameterParser := regexp.MustCompile(`.*\([^\)]*\)`) - matches := parameterParser.FindStringSubmatch(value) - if len(matches) == 0 { - return false - } - parameterString := matches[0] - optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option)) - return len(optionParser.FindStringSubmatch(parameterString)) > 0 -} - -func postProcessTokens(tokens []*token) []*token { - var postProcessedTokens = make([]*token, 0) - - skipNextToken := false - - for index := range tokens { - if skipNextToken { - skipNextToken = false - } else { - postProcessedTokens, skipNextToken = handleToken(tokens, index, postProcessedTokens) - } - } - - return postProcessedTokens -} - -func tokenIsOpType(token *token, opType *operationType) bool { - return token.TokenType == operationToken && token.Operation.OperationType == opType -} - -func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) { - skipNextToken = false - currentToken := tokens[index] - - log.Debug("processing %v", currentToken.toString(true)) - - if currentToken.TokenType == traverseArrayCollect { - // `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator - //need to put a traverse array then a collect currentToken - // do this by adding traverse then converting currentToken to collect - - log.Debug(" adding self") - op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) - - log.Debug(" adding traverse array") - op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) - - currentToken = &token{TokenType: openCollect} - - } - - if tokenIsOpType(currentToken, createMapOpType) { - log.Debugf("tokenIsOpType: createMapOpType") - // check the previous token is '[', means we are slice, but dont have a first number - if tokens[index-1].TokenType == traverseArrayCollect { - log.Debugf("previous token is : traverseArrayOpType") - // need to put the number 0 before this token, as that is implied - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")}) - } - } - - if index != len(tokens)-1 && currentToken.AssignOperation != nil && - tokenIsOpType(tokens[index+1], assignOpType) { - log.Debug(" its an update assign") - currentToken.Operation = currentToken.AssignOperation - currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign - skipNextToken = true - } - - log.Debug(" adding token to the fixed list") - postProcessedTokens = append(postProcessedTokens, currentToken) - - if tokenIsOpType(currentToken, createMapOpType) { - log.Debugf("tokenIsOpType: createMapOpType") - // check the next token is ']', means we are slice, but dont have a second number - if index != len(tokens)-1 && tokens[index+1].TokenType == closeCollect { - log.Debugf("next token is : closeCollect") - // need to put the number 0 before this token, as that is implied - lengthOp := &Operation{OperationType: lengthOpType} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: lengthOp}) - } - } - - if index != len(tokens)-1 && - ((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) || - (currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) { - log.Debug(" adding empty") - op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) - } - - if index != len(tokens)-1 && currentToken.CheckForPostTraverse && - - (tokenIsOpType(tokens[index+1], traversePathOpType) || - (tokens[index+1].TokenType == traverseArrayCollect)) { - log.Debug(" adding pipe because the next thing is traverse") - op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) - } - if index != len(tokens)-1 && currentToken.CheckForPostTraverse && - tokens[index+1].TokenType == openCollect { - - log.Debug(" adding traverArray because next is opencollect") - op := &Operation{OperationType: traverseArrayOpType} - postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) - } - return postProcessedTokens, skipNextToken -} diff --git a/external/yq/pkg/yqlib/lexer_participle.go b/external/yq/pkg/yqlib/lexer_participle.go deleted file mode 100755 index 04f6213..0000000 --- a/external/yq/pkg/yqlib/lexer_participle.go +++ /dev/null @@ -1,554 +0,0 @@ -package yqlib - -import ( - "strconv" - "strings" - - "github.com/alecthomas/participle/v2/lexer" -) - -var participleYqRules = []*participleYqRule{ - {"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0}, - {"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0}, - {"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0}, - - {"OpenBracket", `\(`, literalToken(openBracket, false), 0}, - {"CloseBracket", `\)`, literalToken(closeBracket, true), 0}, - {"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0}, - - {"OpenCollect", `\[`, literalToken(openCollect, false), 0}, - {"CloseCollect", `\]\??`, literalToken(closeCollect, true), 0}, - - {"OpenCollectObject", `\{`, literalToken(openCollectObject, false), 0}, - {"CloseCollectObject", `\}`, literalToken(closeCollectObject, true), 0}, - - {"RecursiveDecentIncludingKeys", `\.\.\.`, recursiveDecentOpToken(true), 0}, - {"RecursiveDecent", `\.\.`, recursiveDecentOpToken(false), 0}, - - {"GetVariable", `\$[a-zA-Z_\-0-9]+`, getVariableOpToken(), 0}, - {"AsignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0}, - {"AsignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0}, - - {"CreateMap", `:\s*`, opToken(createMapOpType), 0}, - simpleOp("length", lengthOpType), - simpleOp("line", lineOpType), - simpleOp("column", columnOpType), - simpleOp("eval", evalOpType), - - {"MapValues", `map_?values`, opToken(mapValuesOpType), 0}, - simpleOp("map", mapOpType), - simpleOp("pick", pickOpType), - - {"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0}, - {"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0}, - - simpleOp("format_datetime", formatDateTimeOpType), - simpleOp("now", nowOpType), - simpleOp("tz", tzOpType), - simpleOp("with_dtf", withDtFormatOpType), - simpleOp("error", errorOpType), - simpleOp("sortKeys", sortKeysOpType), - simpleOp("sort_?keys", sortKeysOpType), - - {"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0}, - - {"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0}, - {"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0}, - {"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0}, - - {"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlInputFormat), 0}, - {"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlOutputFormat, 2), 0}, - - {"JSONEncode", `to_?json`, encodeWithIndent(JSONOutputFormat, 2), 0}, - {"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONOutputFormat, 0), 0}, - - {"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesInputFormat), 0}, - {"PropsEncode", `to_?props|@props`, encodeWithIndent(PropsOutputFormat, 2), 0}, - - {"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLInputFormat), 0}, - {"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0}, - {"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0}, - - {"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVObjectInputFormat), 0}, - {"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0}, - - {"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVObjectInputFormat), 0}, - {"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0}, - - {"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0}, - {"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0}, - - {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0}, - - {"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0}, - - {"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0}, - - {"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0}, - - {"LoadYaml", `load`, loadOp(NewYamlDecoder(ConfiguredYamlPreferences), false), 0}, - - {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, - - simpleOp("select", selectOpType), - simpleOp("has", hasOpType), - simpleOp("unique_?by", uniqueByOpType), - simpleOp("unique", uniqueOpType), - - simpleOp("group_?by", groupByOpType), - simpleOp("explode", explodeOpType), - simpleOp("or", orOpType), - simpleOp("and", andOpType), - simpleOp("not", notOpType), - simpleOp("ireduce", reduceOpType), - - simpleOp("join", joinStringOpType), - simpleOp("sub", subStringOpType), - simpleOp("match", matchOpType), - simpleOp("capture", captureOpType), - simpleOp("test", testOpType), - - simpleOp("sort_?by", sortByOpType), - simpleOp("sort", sortOpType), - - simpleOp("reverse", reverseOpType), - - simpleOp("any_c", anyConditionOpType), - simpleOp("any", anyOpType), - - simpleOp("all_c", allConditionOpType), - simpleOp("all", allOpType), - - simpleOp("contains", containsOpType), - simpleOp("split", splitStringOpType), - simpleOp("parent", getParentOpType), - - simpleOp("keys", keysOpType), - simpleOp("key", getKeyOpType), - simpleOp("is_?key", isKeyOpType), - - simpleOp("file_?name|fileName", getFilenameOpType), - simpleOp("file_?index|fileIndex|fi", getFileIndexOpType), - simpleOp("path", getPathOpType), - simpleOp("set_?path", setPathOpType), - simpleOp("del_?paths", delPathsOpType), - - simpleOp("to_?entries|toEntries", toEntriesOpType), - simpleOp("from_?entries|fromEntries", fromEntriesOpType), - simpleOp("with_?entries|withEntries", withEntriesOpType), - - simpleOp("with", withOpType), - - simpleOp("collect", collectOpType), - simpleOp("del", deleteChildOpType), - - assignableOp("style", getStyleOpType, assignStyleOpType), - assignableOp("tag|type", getTagOpType, assignTagOpType), - assignableOp("anchor", getAnchorOpType, assignAnchorOpType), - assignableOp("alias", getAliasOpType, assignAliasOpType), - - {"ALL_COMMENTS", `comments\s*=`, assignAllCommentsOp(false), 0}, - {"ALL_COMMENTS_ASSIGN_RELATIVE", `comments\s*\|=`, assignAllCommentsOp(true), 0}, - - {"Block", `;`, opToken(blockOpType), 0}, - {"Alternative", `\/\/`, opToken(alternativeOpType), 0}, - - {"DocumentIndex", `documentIndex|document_?index|di`, opToken(getDocumentIndexOpType), 0}, - - {"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0}, - {"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0}, - simpleOp("trim", trimOpType), - - {"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0}, - {"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0}, - {"FloatValue", `-?\d+(\.\d+)`, floatValue(), 0}, - - {"NumberValue", `-?\d+`, numberValue(), 0}, - - {"TrueBooleanValue", `[Tt][Rr][Uu][Ee]`, booleanValue(true), 0}, - {"FalseBooleanValue", `[Ff][Aa][Ll][Ss][Ee]`, booleanValue(false), 0}, - - {"NullValue", `[Nn][Uu][Ll][Ll]|~`, nullValue(), 0}, - - {"QuotedStringValue", `"([^"\\]*(\\.[^"\\]*)*)"`, stringValue(), 0}, - - {"StrEnvOp", `strenv\([^\)]+\)`, envOp(true), 0}, - {"EnvOp", `env\([^\)]+\)`, envOp(false), 0}, - - {"EnvSubstWithOptions", `envsubst\((ne|nu|ff| |,)+\)`, envSubstWithOptions(), 0}, - simpleOp("envsubst", envsubstOpType), - - {"Equals", `\s*==\s*`, opToken(equalsOpType), 0}, - {"NotEquals", `\s*!=\s*`, opToken(notEqualsOpType), 0}, - - {"GreaterThanEquals", `\s*>=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}), 0}, - {"LessThanEquals", `\s*<=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}), 0}, - - {"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0}, - {"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0}, - - {"AssignRelative", `\|=[c]*`, assignOpToken(true), 0}, - {"Assign", `=[c]*`, assignOpToken(false), 0}, - - {`whitespace`, `[ \t\n]+`, nil, 0}, - - {"WrappedPathElement", `\."[^ "]+"\??`, pathToken(true), 0}, - {"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`, pathToken(false), 0}, - {"Pipe", `\|`, opToken(pipeOpType), 0}, - {"Self", `\.`, opToken(selfReferenceOpType), 0}, - - {"Union", `,`, opToken(unionOpType), 0}, - - {"MultiplyAssign", `\*=[\+|\?cdn]*`, multiplyWithPrefs(multiplyAssignOpType), 0}, - {"Multiply", `\*[\+|\?cdn]*`, multiplyWithPrefs(multiplyOpType), 0}, - - {"AddAssign", `\+=`, opToken(addAssignOpType), 0}, - {"Add", `\+`, opToken(addOpType), 0}, - - {"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0}, - {"Subtract", `\-`, opToken(subtractOpType), 0}, -} - -type yqAction func(lexer.Token) (*token, error) - -type participleYqRule struct { - Name string - Pattern string - CreateYqToken yqAction - ParticipleTokenType lexer.TokenType -} - -type participleLexer struct { - lexerDefinition lexer.StringDefinition -} - -func simpleOp(name string, opType *operationType) *participleYqRule { - return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0} -} - -func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule { - return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0} -} - -func newParticipleLexer() expressionTokeniser { - simpleRules := make([]lexer.SimpleRule, len(participleYqRules)) - for i, yqRule := range participleYqRules { - simpleRules[i] = lexer.SimpleRule{Name: yqRule.Name, Pattern: yqRule.Pattern} - } - lexerDefinition := lexer.MustSimple(simpleRules) - symbols := lexerDefinition.Symbols() - - for _, yqRule := range participleYqRules { - yqRule.ParticipleTokenType = symbols[yqRule.Name] - } - - return &participleLexer{lexerDefinition} -} - -func pathToken(wrapped bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - prefs := traversePreferences{} - - if value[len(value)-1:] == "?" { - prefs.OptionalTraverse = true - value = value[:len(value)-1] - } - - value = value[1:] - if wrapped { - value = unwrap(value) - } - log.Debug("PathToken %v", value) - op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil - } -} - -func recursiveDecentOpToken(includeMapKeys bool) yqAction { - prefs := recursiveDescentPreferences{ - RecurseArray: true, - TraversePreferences: traversePreferences{ - DontFollowAlias: true, - IncludeMapKeys: includeMapKeys, - }, - } - return opTokenWithPrefs(recursiveDescentOpType, nil, prefs) -} - -func opTokenWithPrefs(opType *operationType, assignOpType *operationType, preferences interface{}) yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - op := &Operation{OperationType: opType, Value: opType.Type, StringValue: value, Preferences: preferences} - var assign *Operation - if assignOpType != nil { - assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences} - } - return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil - } -} - -func expressionOpToken(expression string) yqAction { - return func(rawToken lexer.Token) (*token, error) { - prefs := expressionOpPreferences{expression: expression} - expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs} - return &token{TokenType: operationToken, Operation: expressionOp}, nil - } -} - -func flattenWithDepth() yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - var depth, errParsingInt = extractNumberParameter(value) - if errParsingInt != nil { - return nil, errParsingInt - } - - prefs := flattenPreferences{depth: depth} - op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil - } -} - -func assignAllCommentsOp(updateAssign bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - log.Debug("assignAllCommentsOp %v", rawToken.Value) - value := rawToken.Value - op := &Operation{ - OperationType: assignCommentOpType, - Value: assignCommentOpType.Type, - StringValue: value, - UpdateAssign: updateAssign, - Preferences: commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true}, - } - return &token{TokenType: operationToken, Operation: op}, nil - } -} - -func assignOpToken(updateAssign bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - log.Debug("assignOpToken %v", rawToken.Value) - value := rawToken.Value - prefs := assignPreferences{DontOverWriteAnchor: true} - if strings.Contains(value, "c") { - prefs.ClobberCustomTags = true - } - op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil - } -} - -func booleanValue(val bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - return &token{TokenType: operationToken, Operation: createValueOperation(val, rawToken.Value)}, nil - } -} - -func nullValue() yqAction { - return func(rawToken lexer.Token) (*token, error) { - return &token{TokenType: operationToken, Operation: createValueOperation(nil, rawToken.Value)}, nil - } -} - -func stringValue() yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := unwrap(rawToken.Value) - value = strings.ReplaceAll(value, "\\\"", "\"") - return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil - } -} - -func envOp(strenv bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - preferences := envOpPreferences{} - - if strenv { - // strenv( ) - value = value[7 : len(value)-1] - preferences.StringValue = true - } else { - //env( ) - value = value[4 : len(value)-1] - } - - envOperation := createValueOperation(value, value) - envOperation.OperationType = envOpType - envOperation.Preferences = preferences - - return &token{TokenType: operationToken, Operation: envOperation}, nil - } -} - -func envSubstWithOptions() yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - noEmpty := hasOptionParameter(value, "ne") - noUnset := hasOptionParameter(value, "nu") - failFast := hasOptionParameter(value, "ff") - envsubstOpType.Type = "ENVSUBST" - prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast} - if noEmpty { - envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY" - } - if noUnset { - envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET" - } - - op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil - } -} - -func multiplyWithPrefs(op *operationType) yqAction { - return func(rawToken lexer.Token) (*token, error) { - prefs := multiplyPreferences{} - prefs.AssignPrefs = assignPreferences{} - options := rawToken.Value - if strings.Contains(options, "+") { - prefs.AppendArrays = true - } - if strings.Contains(options, "?") { - prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} - } - if strings.Contains(options, "n") { - prefs.AssignPrefs.OnlyWriteNull = true - } - if strings.Contains(options, "d") { - prefs.DeepMergeArrays = true - } - if strings.Contains(options, "c") { - prefs.AssignPrefs.ClobberCustomTags = true - } - prefs.TraversePrefs.DontFollowAlias = true - op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil - } - -} - -func getVariableOpToken() yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - - value = value[1:] - - getVarOperation := createValueOperation(value, value) - getVarOperation.OperationType = getVariableOpType - - return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil - } -} - -func hexValue() yqAction { - return func(rawToken lexer.Token) (*token, error) { - var originalString = rawToken.Value - var numberString = originalString[2:] - log.Debugf("numberString: %v", numberString) - var number, errParsingInt = strconv.ParseInt(numberString, 16, 64) - if errParsingInt != nil { - return nil, errParsingInt - } - - return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil - } -} - -func floatValue() yqAction { - return func(rawToken lexer.Token) (*token, error) { - var numberString = rawToken.Value - var number, errParsingInt = strconv.ParseFloat(numberString, 64) - if errParsingInt != nil { - return nil, errParsingInt - } - return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil - } -} - -func numberValue() yqAction { - return func(rawToken lexer.Token) (*token, error) { - var numberString = rawToken.Value - var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) - if errParsingInt != nil { - return nil, errParsingInt - } - - return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil - } -} - -func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction { - return func(rawToken lexer.Token) (*token, error) { - value := rawToken.Value - var indent, errParsingInt = extractNumberParameter(value) - if errParsingInt != nil { - return nil, errParsingInt - } - - prefs := encoderPreferences{format: outputFormat, indent: indent} - op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs} - return &token{TokenType: operationToken, Operation: op}, nil - } -} - -func encodeWithIndent(outputFormat PrinterOutputFormat, indent int) yqAction { - prefs := encoderPreferences{format: outputFormat, indent: indent} - return opTokenWithPrefs(encodeOpType, nil, prefs) -} - -func decodeOp(inputFormat InputFormat) yqAction { - prefs := decoderPreferences{format: inputFormat} - return opTokenWithPrefs(decodeOpType, nil, prefs) -} - -func loadOp(decoder Decoder, loadAsString bool) yqAction { - prefs := loadPrefs{decoder: decoder, loadAsString: loadAsString} - return opTokenWithPrefs(loadOpType, nil, prefs) -} - -func opToken(op *operationType) yqAction { - return opTokenWithPrefs(op, nil, nil) -} - -func literalToken(tt tokenType, checkForPost bool) yqAction { - return func(rawToken lexer.Token) (*token, error) { - return &token{TokenType: tt, CheckForPostTraverse: checkForPost, Match: rawToken.Value}, nil - } -} - -func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule { - for _, yqRule := range participleYqRules { - if yqRule.ParticipleTokenType == rawToken.Type { - return yqRule - } - } - return &participleYqRule{} -} - -func (p *participleLexer) Tokenise(expression string) ([]*token, error) { - myLexer, err := p.lexerDefinition.LexString("", expression) - if err != nil { - return nil, err - } - tokens := make([]*token, 0) - - for { - rawToken, e := myLexer.Next() - if e != nil { - return nil, e - } else if rawToken.Type == lexer.EOF { - return postProcessTokens(tokens), nil - } - - definition := p.getYqDefinition(rawToken) - if definition.CreateYqToken != nil { - token, e := definition.CreateYqToken(rawToken) - if e != nil { - return nil, e - } - tokens = append(tokens, token) - } - - } - -} diff --git a/external/yq/pkg/yqlib/lexer_participle_test.go b/external/yq/pkg/yqlib/lexer_participle_test.go deleted file mode 100755 index a1defe4..0000000 --- a/external/yq/pkg/yqlib/lexer_participle_test.go +++ /dev/null @@ -1,658 +0,0 @@ -package yqlib - -import ( - "testing" - - "github.com/alecthomas/repr" - "github.com/mikefarah/yq/v4/test" - yaml "gopkg.in/yaml.v3" -) - -type participleLexerScenario struct { - expression string - tokens []*token -} - -var participleLexerScenarios = []participleLexerScenario{ - { - expression: ".[:3]", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: selfReferenceOpType, - StringValue: "SELF", - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traverseArrayOpType, - StringValue: "TRAVERSE_ARRAY", - }, - }, - { - TokenType: openCollect, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: valueOpType, - Value: 0, - StringValue: "0", - CandidateNode: &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "0", - }, - }, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: createMapOpType, - Value: "CREATE_MAP", - StringValue: ":", - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: valueOpType, - Value: 3, - StringValue: "3", - CandidateNode: &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.Kind(8), - Tag: "!!int", - Value: "3", - }, - }, - }, - }, - { - TokenType: closeCollect, - CheckForPostTraverse: true, - Match: "]", - }, - }, - }, - { - expression: ".[-2:]", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: selfReferenceOpType, - StringValue: "SELF", - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traverseArrayOpType, - StringValue: "TRAVERSE_ARRAY", - }, - }, - { - TokenType: openCollect, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: valueOpType, - Value: -2, - StringValue: "-2", - CandidateNode: &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "-2", - }, - }, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: createMapOpType, - Value: "CREATE_MAP", - StringValue: ":", - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: lengthOpType, - }, - }, - { - TokenType: closeCollect, - CheckForPostTraverse: true, - Match: "]", - }, - }, - }, - { - expression: ".a", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: ".a.b", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: shortPipeOpType, - Value: "PIPE", - StringValue: ".", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b", - StringValue: "b", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: ".a.b?", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: shortPipeOpType, - Value: "PIPE", - StringValue: ".", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b", - StringValue: "b", - Preferences: traversePreferences{ - OptionalTraverse: true, - }, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: `.a."b?"`, - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: shortPipeOpType, - Value: "PIPE", - StringValue: ".", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b?", - StringValue: "b?", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: ` .a ."b?"`, - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: shortPipeOpType, - Value: "PIPE", - StringValue: ".", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b?", - StringValue: "b?", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: `.a | .b`, - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: pipeOpType, - Value: "PIPE", - StringValue: "|", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b", - StringValue: "b", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: "(.a)", - tokens: []*token{ - { - TokenType: openBracket, - Match: "(", - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: closeBracket, - Match: ")", - CheckForPostTraverse: true, - }, - }, - }, - { - expression: "..", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: recursiveDescentOpType, - Value: "RECURSIVE_DESCENT", - StringValue: "..", - Preferences: recursiveDescentPreferences{ - RecurseArray: true, - TraversePreferences: traversePreferences{ - DontFollowAlias: true, - IncludeMapKeys: false, - }, - }, - }, - }, - }, - }, - { - expression: "...", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: recursiveDescentOpType, - Value: "RECURSIVE_DESCENT", - StringValue: "...", - Preferences: recursiveDescentPreferences{ - RecurseArray: true, - TraversePreferences: traversePreferences{ - DontFollowAlias: true, - IncludeMapKeys: true, - }, - }, - }, - }, - }, - }, - { - expression: ".a,.b", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "a", - StringValue: "a", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: unionOpType, - Value: "UNION", - StringValue: ",", - Preferences: nil, - }, - }, - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: traversePathOpType, - Value: "b", - StringValue: "b", - Preferences: traversePreferences{}, - }, - CheckForPostTraverse: true, - }, - }, - }, - { - expression: "map_values", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: mapValuesOpType, - Value: "MAP_VALUES", - StringValue: "map_values", - Preferences: nil, - }, - }, - }, - }, - { - expression: "mapvalues", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: mapValuesOpType, - Value: "MAP_VALUES", - StringValue: "mapvalues", - Preferences: nil, - }, - }, - }, - }, - { - expression: "flatten(3)", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: flattenOpType, - Value: "FLATTEN_BY", - StringValue: "flatten(3)", - Preferences: flattenPreferences{depth: 3}, - }, - }, - }, - }, - { - expression: "flatten", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: flattenOpType, - Value: "FLATTEN_BY", - StringValue: "flatten", - Preferences: flattenPreferences{depth: -1}, - }, - }, - }, - }, - { - expression: "length", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: lengthOpType, - Value: "LENGTH", - StringValue: "length", - Preferences: nil, - }, - }, - }, - }, - { - expression: "format_datetime", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: formatDateTimeOpType, - Value: "FORMAT_DATE_TIME", - StringValue: "format_datetime", - Preferences: nil, - }, - }, - }, - }, - { - expression: "to_yaml(3)", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: encodeOpType, - Value: "ENCODE", - StringValue: "to_yaml(3)", - Preferences: encoderPreferences{ - format: YamlOutputFormat, - indent: 3, - }, - }, - }, - }, - }, - { - expression: "tojson(2)", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: encodeOpType, - Value: "ENCODE", - StringValue: "tojson(2)", - Preferences: encoderPreferences{ - format: JSONOutputFormat, - indent: 2, - }, - }, - }, - }, - }, - { - expression: "@yaml", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: encodeOpType, - Value: "ENCODE", - StringValue: "@yaml", - Preferences: encoderPreferences{ - format: YamlOutputFormat, - indent: 2, - }, - }, - }, - }, - }, - { - expression: "to_props", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: encodeOpType, - Value: "ENCODE", - StringValue: "to_props", - Preferences: encoderPreferences{ - format: PropsOutputFormat, - indent: 2, - }, - }, - }, - }, - }, - { - expression: "@base64d", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: decodeOpType, - Value: "DECODE", - StringValue: "@base64d", - Preferences: decoderPreferences{ - format: Base64InputFormat, - }, - }, - }, - }, - }, - { - expression: "@base64", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: encodeOpType, - Value: "ENCODE", - StringValue: "@base64", - Preferences: encoderPreferences{ - format: Base64OutputFormat, - }, - }, - }, - }, - }, - { - expression: "@yamld", - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: decodeOpType, - Value: "DECODE", - StringValue: "@yamld", - Preferences: decoderPreferences{ - format: YamlInputFormat, - }, - }, - }, - }, - }, - { - expression: `"string with a \""`, - tokens: []*token{ - { - TokenType: operationToken, - Operation: &Operation{ - OperationType: valueOpType, - Value: `string with a "`, - StringValue: `string with a "`, - Preferences: nil, - CandidateNode: &CandidateNode{ - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: `string with a "`, - }, - }, - }, - }, - }, - }, -} - -func TestParticipleLexer(t *testing.T) { - lexer := newParticipleLexer() - - for _, scenario := range participleLexerScenarios { - actual, err := lexer.Tokenise(scenario.expression) - if err != nil { - t.Error(err) - } else { - test.AssertResultWithContext(t, repr.String(scenario.tokens, repr.Indent(" ")), repr.String(actual, repr.Indent(" ")), scenario.expression) - } - - } -} diff --git a/external/yq/pkg/yqlib/lib.go b/external/yq/pkg/yqlib/lib.go deleted file mode 100755 index 6ea1300..0000000 --- a/external/yq/pkg/yqlib/lib.go +++ /dev/null @@ -1,497 +0,0 @@ -// Use the top level Evaluator or StreamEvaluator to evaluate expressions and return matches. -package yqlib - -import ( - "bytes" - "container/list" - "fmt" - "math" - "strconv" - "strings" - - logging "gopkg.in/op/go-logging.v1" - yaml "gopkg.in/yaml.v3" -) - -var ExpressionParser ExpressionParserInterface - -func InitExpressionParser() { - if ExpressionParser == nil { - ExpressionParser = newExpressionParser() - } -} - -var log = logging.MustGetLogger("yq-lib") - -var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""` - -// GetLogger returns the yq logger instance. -func GetLogger() *logging.Logger { - return log -} - -type operationType struct { - Type string - NumArgs uint // number of arguments to the op - Precedence uint - Handler operatorHandler -} - -var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator} -var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator} -var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator} - -var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator} - -var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator} - -var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator} - -var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator} -var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator} -var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator} - -var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator} -var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator} -var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: assignVariableOperator} -var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator} -var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator} -var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator} -var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator} - -var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator} -var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator} - -var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator} -var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator} -var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator} - -var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator} -var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator} - -var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator} - -// createmap needs to be above union, as we use union to build the components of the objects -var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator} - -var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator} - -var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator} -var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator} -var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator} - -var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator} - -var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator} -var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator} -var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator} -var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator} -var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator} -var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator} - -var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime} -var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat} -var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp} -var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp} - -var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator} -var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator} - -var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator} -var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator} -var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator} -var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator} -var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator} - -var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator} -var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator} -var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator} - -var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator} - -var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator} -var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator} -var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator} -var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator} - -var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator} -var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator} -var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator} - -var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator} -var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator} -var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator} -var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator} -var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator} -var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator} - -var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator} -var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator} -var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator} - -var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator} -var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator} -var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator} -var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator} - -var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator} - -var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator} -var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator} -var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator} -var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator} -var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator} -var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator} -var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator} -var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator} - -var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 50, Handler: loadYamlOperator} - -var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator} - -var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} -var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator} -var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator} - -var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator} -var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator} -var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator} -var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator} -var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator} - -var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator} - -var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator} - -var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator} -var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator} -var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique} -var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy} -var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy} -var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp} -var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator} - -type Operation struct { - OperationType *operationType - Value interface{} - StringValue string - CandidateNode *CandidateNode // used for Value Path elements - Preferences interface{} - UpdateAssign bool // used for assign ops, when true it means we evaluate the rhs given the lhs -} - -func recurseNodeArrayEqual(lhs *yaml.Node, rhs *yaml.Node) bool { - if len(lhs.Content) != len(rhs.Content) { - return false - } - - for index := 0; index < len(lhs.Content); index = index + 1 { - if !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) { - return false - } - } - return true -} - -func findInArray(array *yaml.Node, item *yaml.Node) int { - - for index := 0; index < len(array.Content); index = index + 1 { - if recursiveNodeEqual(array.Content[index], item) { - return index - } - } - return -1 -} - -func findKeyInMap(dataMap *yaml.Node, item *yaml.Node) int { - - for index := 0; index < len(dataMap.Content); index = index + 2 { - if recursiveNodeEqual(dataMap.Content[index], item) { - return index - } - } - return -1 -} - -func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool { - if len(lhs.Content) != len(rhs.Content) { - return false - } - - for index := 0; index < len(lhs.Content); index = index + 2 { - key := lhs.Content[index] - value := lhs.Content[index+1] - - indexInRHS := findInArray(rhs, key) - - if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) { - return false - } - } - return true -} - -func guessTagFromCustomType(node *yaml.Node) string { - if strings.HasPrefix(node.Tag, "!!") { - return node.Tag - } else if node.Value == "" { - log.Warning("node has no value to guess the type with") - return node.Tag - } - dataBucket, errorReading := parseSnippet(node.Value) - - if errorReading != nil { - log.Warning("could not guess underlying tag type %v", errorReading) - return node.Tag - } - guessedTag := unwrapDoc(dataBucket).Tag - log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag) - return guessedTag -} - -func parseSnippet(value string) (*yaml.Node, error) { - if value == "" { - return &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!null", - }, nil - } - decoder := NewYamlDecoder(ConfiguredYamlPreferences) - err := decoder.Init(strings.NewReader(value)) - if err != nil { - return nil, err - } - parsedNode, err := decoder.Decode() - if err != nil { - return nil, err - } - if len(parsedNode.Node.Content) == 0 { - return nil, fmt.Errorf("bad data") - } - return unwrapDoc(parsedNode.Node), err -} - -func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { - if lhs.Kind != rhs.Kind { - return false - } - - if lhs.Kind == yaml.ScalarNode { - //process custom tags of scalar nodes. - //dont worry about matching tags of maps or arrays. - - lhsTag := guessTagFromCustomType(lhs) - rhsTag := guessTagFromCustomType(rhs) - - if lhsTag != rhsTag { - return false - } - } - - if lhs.Tag == "!!null" { - return true - - } else if lhs.Kind == yaml.ScalarNode { - return lhs.Value == rhs.Value - } else if lhs.Kind == yaml.SequenceNode { - return recurseNodeArrayEqual(lhs, rhs) - } else if lhs.Kind == yaml.MappingNode { - return recurseNodeObjectEqual(lhs, rhs) - } - return false -} - -func deepCloneContent(content []*yaml.Node) []*yaml.Node { - clonedContent := make([]*yaml.Node, len(content)) - for i, child := range content { - clonedContent[i] = deepClone(child) - } - return clonedContent -} - -func deepCloneNoContent(node *yaml.Node) *yaml.Node { - return deepCloneWithOptions(node, false) -} -func deepClone(node *yaml.Node) *yaml.Node { - return deepCloneWithOptions(node, true) -} - -func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node { - if node == nil { - return nil - } - var clonedContent []*yaml.Node - if cloneContent { - clonedContent = deepCloneContent(node.Content) - } - return &yaml.Node{ - Content: clonedContent, - Kind: node.Kind, - Style: node.Style, - Tag: node.Tag, - Value: node.Value, - Anchor: node.Anchor, - Alias: deepClone(node.Alias), - HeadComment: node.HeadComment, - LineComment: node.LineComment, - FootComment: node.FootComment, - Line: node.Line, - Column: node.Column, - } -} - -// yaml numbers can be hex encoded... -func parseInt64(numberString string) (string, int64, error) { - if strings.HasPrefix(numberString, "0x") || - strings.HasPrefix(numberString, "0X") { - num, err := strconv.ParseInt(numberString[2:], 16, 64) - return "0x%X", num, err - } - num, err := strconv.ParseInt(numberString, 10, 64) - return "%v", num, err -} - -func parseInt(numberString string) (int, error) { - var err error - var parsed int64 - if strings.HasPrefix(numberString, "0x") || - strings.HasPrefix(numberString, "0X") { - parsed, err = strconv.ParseInt(numberString[2:], 16, 64) - } else { - parsed, err = strconv.ParseInt(numberString, 10, 64) - } - - if err != nil { - return 0, err - } else if parsed > math.MaxInt || parsed < math.MinInt { - return 0, fmt.Errorf("%v is not within [%v, %v]", parsed, math.MinInt, math.MaxInt) - } - - return int(parsed), err -} - -func createStringScalarNode(stringValue string) *yaml.Node { - var node = &yaml.Node{Kind: yaml.ScalarNode} - node.Value = stringValue - node.Tag = "!!str" - return node -} - -func createScalarNode(value interface{}, stringValue string) *yaml.Node { - var node = &yaml.Node{Kind: yaml.ScalarNode} - node.Value = stringValue - - switch value.(type) { - case float32, float64: - node.Tag = "!!float" - case int, int64, int32: - node.Tag = "!!int" - case bool: - node.Tag = "!!bool" - case string: - node.Tag = "!!str" - case nil: - node.Tag = "!!null" - } - return node -} - -func headAndLineComment(node *yaml.Node) string { - return headComment(node) + lineComment(node) -} - -func headComment(node *yaml.Node) string { - return strings.Replace(node.HeadComment, "#", "", 1) -} - -func lineComment(node *yaml.Node) string { - return strings.Replace(node.LineComment, "#", "", 1) -} - -func footComment(node *yaml.Node) string { - return strings.Replace(node.FootComment, "#", "", 1) -} - -func createValueOperation(value interface{}, stringValue string) *Operation { - var node = createScalarNode(value, stringValue) - - return &Operation{ - OperationType: valueOpType, - Value: value, - StringValue: stringValue, - CandidateNode: &CandidateNode{Node: node}, - } -} - -// debugging purposes only -func (p *Operation) toString() string { - if p == nil { - return "OP IS NIL" - } - if p.OperationType == traversePathOpType { - return fmt.Sprintf("%v", p.Value) - } else if p.OperationType == selfReferenceOpType { - return "SELF" - } else if p.OperationType == valueOpType { - return fmt.Sprintf("%v (%T)", p.Value, p.Value) - } else { - return fmt.Sprintf("%v", p.OperationType.Type) - } -} - -// use for debugging only -func NodesToString(collection *list.List) string { - if !log.IsEnabledFor(logging.DEBUG) { - return "" - } - - result := fmt.Sprintf("%v results\n", collection.Len()) - for el := collection.Front(); el != nil; el = el.Next() { - result = result + "\n" + NodeToString(el.Value.(*CandidateNode)) - } - return result -} - -func NodeToString(node *CandidateNode) string { - if !log.IsEnabledFor(logging.DEBUG) { - return "" - } - value := node.Node - if value == nil { - return "-- nil --" - } - buf := new(bytes.Buffer) - encoder := yaml.NewEncoder(buf) - errorEncoding := encoder.Encode(value) - if errorEncoding != nil { - log.Error("Error debugging node, %v", errorEncoding.Error()) - } - errorClosingEncoder := encoder.Close() - if errorClosingEncoder != nil { - log.Error("Error closing encoder: ", errorClosingEncoder.Error()) - } - tag := value.Tag - if value.Kind == yaml.DocumentNode { - tag = "doc" - } else if value.Kind == yaml.AliasNode { - tag = "alias" - } - return fmt.Sprintf(`D%v, P%v, (%v)::%v`, node.Document, node.Path, tag, buf.String()) -} - -func KindString(kind yaml.Kind) string { - switch kind { - case yaml.ScalarNode: - return "ScalarNode" - case yaml.SequenceNode: - return "SequenceNode" - case yaml.MappingNode: - return "MappingNode" - case yaml.DocumentNode: - return "DocumentNode" - case yaml.AliasNode: - return "AliasNode" - default: - return "unknown!" - } -} diff --git a/external/yq/pkg/yqlib/lib_test.go b/external/yq/pkg/yqlib/lib_test.go deleted file mode 100755 index fb3d5c7..0000000 --- a/external/yq/pkg/yqlib/lib_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package yqlib - -import ( - "testing" - - "github.com/mikefarah/yq/v4/test" - yaml "gopkg.in/yaml.v3" -) - -func TestGetLogger(t *testing.T) { - l := GetLogger() - if l != log { - t.Fatal("GetLogger should return the yq logger instance, not a copy") - } -} - -type parseSnippetScenario struct { - snippet string - expected *yaml.Node - expectedError string -} - -var parseSnippetScenarios = []parseSnippetScenario{ - { - snippet: ":", - expectedError: "yaml: did not find expected key", - }, - { - snippet: "", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!null", - }, - }, - { - snippet: "null", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!null", - Value: "null", - Line: 1, - Column: 1, - }, - }, - { - snippet: "3", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: "3", - Line: 1, - Column: 1, - }, - }, - { - snippet: "cat", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: "cat", - Line: 1, - Column: 1, - }, - }, - { - snippet: "3.1", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!float", - Value: "3.1", - Line: 1, - Column: 1, - }, - }, - { - snippet: "true", - expected: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!bool", - Value: "true", - Line: 1, - Column: 1, - }, - }, -} - -func TestParseSnippet(t *testing.T) { - for _, tt := range parseSnippetScenarios { - actual, err := parseSnippet(tt.snippet) - if tt.expectedError != "" { - if err == nil { - t.Errorf("Expected error '%v' but it worked!", tt.expectedError) - } else { - test.AssertResultComplexWithContext(t, tt.expectedError, err.Error(), tt.snippet) - } - return - } - if err != nil { - t.Error(tt.snippet) - t.Error(err) - } - test.AssertResultComplexWithContext(t, tt.expected, actual, tt.snippet) - } -} diff --git a/external/yq/pkg/yqlib/matchKeyString.go b/external/yq/pkg/yqlib/matchKeyString.go deleted file mode 100755 index bfea235..0000000 --- a/external/yq/pkg/yqlib/matchKeyString.go +++ /dev/null @@ -1,58 +0,0 @@ -package yqlib - -func matchKey(name string, pattern string) (matched bool) { - if pattern == "" { - return name == pattern - } - log.Debug("pattern: %v", pattern) - if pattern == "*" { - log.Debug("wild!") - return true - } - return deepMatch(name, pattern) -} - -// deepMatch reports whether the name matches the pattern in linear time. -// Source https://research.swtch.com/glob -func deepMatch(name, pattern string) bool { - px := 0 - nx := 0 - nextPx := 0 - nextNx := 0 - for px < len(pattern) || nx < len(name) { - if px < len(pattern) { - c := pattern[px] - switch c { - default: // ordinary character - if nx < len(name) && name[nx] == c { - px++ - nx++ - continue - } - case '?': // single-character wildcard - if nx < len(name) { - px++ - nx++ - continue - } - case '*': // zero-or-more-character wildcard - // Try to match at nx. - // If that doesn't work out, - // restart at nx+1 next. - nextPx = px - nextNx = nx + 1 - px++ - continue - } - } - // Mismatch. Maybe restart. - if 0 < nextNx && nextNx <= len(name) { - px = nextPx - nx = nextNx - continue - } - return false - } - // Matched all of pattern to all of name. Success. - return true -} diff --git a/external/yq/pkg/yqlib/matchKeyString_test.go b/external/yq/pkg/yqlib/matchKeyString_test.go deleted file mode 100755 index 3cacb4a..0000000 --- a/external/yq/pkg/yqlib/matchKeyString_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package yqlib - -import ( - "strings" - "testing" -) - -func TestDeepMatch(t *testing.T) { - var tests = []struct { - name string - pattern string - ok bool - }{ - {"", "", true}, - {"", "x", false}, - {"x", "", false}, - {"abc", "abc", true}, - {"abc", "*", true}, - {"abc", "*c", true}, - {"abc", "*b", false}, - {"abc", "a*", true}, - {"abc", "b*", false}, - {"a", "a*", true}, - {"a", "*a", true}, - {"axbxcxdxe", "a*b*c*d*e*", true}, - {"axbxcxdxexxx", "a*b*c*d*e*", true}, - {"abxbbxdbxebxczzx", "a*b?c*x", true}, - {"abxbbxdbxebxczzy", "a*b?c*x", false}, - {strings.Repeat("a", 100), "a*a*a*a*b", false}, - {"xxx", "*x", true}, - } - - for _, tt := range tests { - t.Run(tt.name+" "+tt.pattern, func(t *testing.T) { - if want, got := tt.ok, deepMatch(tt.name, tt.pattern); want != got { - t.Errorf("Expected %v got %v", want, got) - } - }) - } -} diff --git a/external/yq/pkg/yqlib/operator_add.go b/external/yq/pkg/yqlib/operator_add.go deleted file mode 100755 index f0281bd..0000000 --- a/external/yq/pkg/yqlib/operator_add.go +++ /dev/null @@ -1,209 +0,0 @@ -package yqlib - -import ( - "fmt" - "strconv" - "strings" - "time" - - yaml "gopkg.in/yaml.v3" -) - -func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { - return &ExpressionNode{Operation: &Operation{OperationType: addOpType}, - LHS: lhs, - RHS: rhs} -} - -func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - return compoundAssignFunction(d, context, expressionNode, createAddOp) -} - -func toNodes(candidate *CandidateNode, lhs *CandidateNode) ([]*yaml.Node, error) { - if candidate.Node.Tag == "!!null" { - return []*yaml.Node{}, nil - } - clone, err := candidate.Copy() - if err != nil { - return nil, err - } - - switch candidate.Node.Kind { - case yaml.SequenceNode: - return clone.Node.Content, nil - default: - if len(lhs.Node.Content) > 0 { - clone.Node.Style = lhs.Node.Content[0].Style - } - return []*yaml.Node{clone.Node}, nil - } - -} - -func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("Add operator") - - return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false) -} - -func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - - lhsNode := lhs.Node - - if lhsNode.Tag == "!!null" { - return lhs.CreateReplacement(rhs.Node), nil - } - - target := lhs.CreateReplacement(&yaml.Node{ - Anchor: lhs.Node.Anchor, - }) - - switch lhsNode.Kind { - case yaml.MappingNode: - if rhs.Node.Kind != yaml.MappingNode { - return nil, fmt.Errorf("%v (%v) cannot be added to a %v (%v)", rhs.Node.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath()) - } - addMaps(target, lhs, rhs) - case yaml.SequenceNode: - if err := addSequences(target, lhs, rhs); err != nil { - return nil, err - } - - case yaml.ScalarNode: - if rhs.Node.Kind != yaml.ScalarNode { - return nil, fmt.Errorf("%v (%v) cannot be added to a %v (%v)", rhs.Node.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath()) - } - target.Node.Kind = yaml.ScalarNode - target.Node.Style = lhsNode.Style - if err := addScalars(context, target, lhsNode, rhs.Node); err != nil { - return nil, err - } - } - return target, nil -} - -func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { - lhsTag := lhs.Tag - rhsTag := guessTagFromCustomType(rhs) - lhsIsCustom := false - if !strings.HasPrefix(lhsTag, "!!") { - // custom tag - we have to have a guess - lhsTag = guessTagFromCustomType(lhs) - lhsIsCustom = true - } - - isDateTime := lhs.Tag == "!!timestamp" - - // if the lhs is a string, it might be a timestamp in a custom format. - if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { - _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) - isDateTime = err == nil - } - - if isDateTime { - return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs) - - } else if lhsTag == "!!str" { - target.Node.Tag = lhs.Tag - target.Node.Value = lhs.Value + rhs.Value - } else if rhsTag == "!!str" { - target.Node.Tag = rhs.Tag - target.Node.Value = lhs.Value + rhs.Value - } else if lhsTag == "!!int" && rhsTag == "!!int" { - format, lhsNum, err := parseInt64(lhs.Value) - if err != nil { - return err - } - _, rhsNum, err := parseInt64(rhs.Value) - if err != nil { - return err - } - sum := lhsNum + rhsNum - target.Node.Tag = lhs.Tag - target.Node.Value = fmt.Sprintf(format, sum) - } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { - lhsNum, err := strconv.ParseFloat(lhs.Value, 64) - if err != nil { - return err - } - rhsNum, err := strconv.ParseFloat(rhs.Value, 64) - if err != nil { - return err - } - sum := lhsNum + rhsNum - if lhsIsCustom { - target.Node.Tag = lhs.Tag - } else { - target.Node.Tag = "!!float" - } - target.Node.Value = fmt.Sprintf("%v", sum) - } else { - return fmt.Errorf("%v cannot be added to %v", lhsTag, rhsTag) - } - return nil -} - -func addDateTimes(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { - - duration, err := time.ParseDuration(rhs.Value) - if err != nil { - return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) - } - - currentTime, err := parseDateTime(layout, lhs.Value) - if err != nil { - return err - } - - newTime := currentTime.Add(duration) - target.Node.Value = newTime.Format(layout) - return nil - -} - -func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { - target.Node.Kind = yaml.SequenceNode - if len(lhs.Node.Content) > 0 { - target.Node.Style = lhs.Node.Style - } - target.Node.Tag = lhs.Node.Tag - - extraNodes, err := toNodes(rhs, lhs) - if err != nil { - return err - } - - target.Node.Content = append(deepCloneContent(lhs.Node.Content), extraNodes...) - return nil - -} - -func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { - lhs := lhsC.Node - rhs := rhsC.Node - - target.Node.Content = make([]*yaml.Node, len(lhs.Content)) - copy(target.Node.Content, lhs.Content) - - for index := 0; index < len(rhs.Content); index = index + 2 { - key := rhs.Content[index] - value := rhs.Content[index+1] - log.Debug("finding %v", key.Value) - indexInLHS := findKeyInMap(target.Node, key) - log.Debug("indexInLhs %v", indexInLHS) - if indexInLHS < 0 { - // not in there, append it - target.Node.Content = append(target.Node.Content, key, value) - } else { - // it's there, replace it - target.Node.Content[indexInLHS+1] = value - } - } - target.Node.Kind = yaml.MappingNode - if len(lhs.Content) > 0 { - target.Node.Style = lhs.Style - } - target.Node.Tag = lhs.Tag -} diff --git a/external/yq/pkg/yqlib/operator_add_test.go b/external/yq/pkg/yqlib/operator_add_test.go deleted file mode 100755 index b745251..0000000 --- a/external/yq/pkg/yqlib/operator_add_test.go +++ /dev/null @@ -1,383 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var addOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `[{a: foo, b: bar}, {a: 1, b: 2}]`, - expression: ".[] | .a + .b", - expected: []string{ - "D0, P[0 a], (!!str)::foobar\n", - "D0, P[1 a], (!!int)::3\n", - }, - }, - { - skipDoc: true, - document: `a: key`, - expression: `. += {"key": "b"}`, - expected: []string{ - "D0, P[], (!!map)::a: key\nkey: b\n", - }, - }, - { - skipDoc: true, - document: `[[c], [b]]`, - expression: `.[] | . += "a"`, - expected: []string{ - "D0, P[0], (!!seq)::[c, a]\n", - "D0, P[1], (!!seq)::[b, a]\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: "(.a + .b) as $x", - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - skipDoc: true, - document: `a: 0`, - expression: ".a += .b.c", - expected: []string{ - "D0, P[], (doc)::a: 0\n", - }, - }, - - { - description: "Concatenate arrays", - document: `{a: [1,2], b: [3,4]}`, - expression: `.a + .b`, - expected: []string{ - "D0, P[a], (!!seq)::[1, 2, 3, 4]\n", - }, - }, - { - description: "Concatenate to existing array", - subdescription: "Note that the styling of `a` is kept.", - document: "a: [1,2]\nb:\n - 3\n - 4", - dontFormatInputForDoc: true, - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: [1, 2, 3, 4]\nb:\n - 3\n - 4\n", - }, - }, - { - skipDoc: true, - expression: `[1] + ([2], [3])`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- 2\n", - "D0, P[], (!!seq)::- 1\n- 3\n", - }, - }, - { - description: "Concatenate null to array", - document: `{a: [1,2]}`, - expression: `.a + null`, - expected: []string{ - "D0, P[a], (!!seq)::[1, 2]\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to empty array", - document: `{a: []}`, - expression: `.a + "cat"`, - expected: []string{ - "D0, P[a], (!!seq)::- cat\n", - }, - }, - { - description: "Append to existing array", - subdescription: "Note that the styling is copied from existing array elements", - dontFormatInputForDoc: true, - document: `a: ['dog']`, - expression: `.a += "cat"`, - expected: []string{ - "D0, P[], (doc)::a: ['dog', 'cat']\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to existing array", - subdescription: "does not modify original", - document: `{a: ['dog'], b: cat}`, - expression: `.a = .a + .b`, - expected: []string{ - "D0, P[], (doc)::{a: ['dog', 'cat'], b: cat}\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to empty array", - document: `a: []`, - expression: `.a += "cat"`, - expected: []string{ - "D0, P[], (doc)::a:\n - cat\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to existing array", - document: `a: [dog]`, - expression: `.a += "cat"`, - expected: []string{ - "D0, P[], (doc)::a: [dog, cat]\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to empty object", - document: `{a: {}}`, - expression: `.a + {"b": "cat"}`, - expected: []string{ - "D0, P[a], (!!map)::b: cat\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to existing object", - document: `{a: {c: dog}}`, - expression: `.a + {"b": "cat"}`, - expected: []string{ - "D0, P[a], (!!map)::{c: dog, b: cat}\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to existing object", - subdescription: "matches stylig", - document: "a:\n c: dog", - expression: `.a + {"b": "cat"}`, - expected: []string{ - "D0, P[a], (!!map)::c: dog\nb: cat\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to empty object in place", - document: `a: {}`, - expression: `.a += {"b": "cat"}`, - expected: []string{ - "D0, P[], (doc)::a:\n b: cat\n", - }, - }, - { - skipDoc: true, - description: "Concatenate to existing object in place", - document: `a: {c: dog}`, - expression: `.a += {"b": "cat"}`, - expected: []string{ - "D0, P[], (doc)::a: {c: dog, b: cat}\n", - }, - }, - { - description: "Add new object to array", - document: `a: [{dog: woof}]`, - expression: `.a + {"cat": "meow"}`, - expected: []string{ - "D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\n", - }, - }, - { - description: "Relative append", - document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`, - expression: `.a[].b += ["mouse"]`, - expected: []string{ - "D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n", - }, - }, - { - description: "String concatenation", - document: `{a: cat, b: meow}`, - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::{a: catmeow, b: meow}\n", - }, - }, - { - description: "String concatenation - str + int", - skipDoc: true, - document: `{a: !cool cat, b: meow}`, - expression: `.a + 3`, - expected: []string{ - "D0, P[a], (!cool)::cat3\n", - }, - }, - { - description: "String concatenation - int + str", - skipDoc: true, - document: `{a: !cool cat, b: meow}`, - expression: `3 + .a`, - expected: []string{ - "D0, P[], (!cool)::3cat\n", - }, - }, - { - description: "Number addition - float", - subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", - document: `{a: 3, b: 4.9}`, - expression: `.a = .a + .b`, - expected: []string{ - "D0, P[], (doc)::{a: 7.9, b: 4.9}\n", - }, - }, - { - description: "Number addition - int", - subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", - document: `{a: 3, b: 4}`, - expression: `.a = .a + .b`, - expected: []string{ - "D0, P[], (doc)::{a: 7, b: 4}\n", - }, - }, - { - description: "Increment numbers", - document: `{a: 3, b: 5}`, - expression: `.[] += 1`, - expected: []string{ - "D0, P[], (doc)::{a: 4, b: 6}\n", - }, - }, - { - description: "Date addition", - subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: 2021-01-01T00:00:00Z`, - expression: `.a += "3h10m"`, - expected: []string{ - "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", - }, - }, - { - description: "Date addition -date only", - skipDoc: true, - document: `a: 2021-01-01`, - expression: `.a += "24h"`, - expected: []string{ - "D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n", - }, - }, - { - description: "Date addition - custom format", - subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", - }, - }, - { - skipDoc: true, - description: "Date addition - custom format", - subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, - expected: []string{ - "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", - }, - }, - { - description: "Add to null", - subdescription: "Adding to null simply returns the rhs", - expression: `null + "cat"`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - description: "Add maps to shallow merge", - subdescription: "Adding objects together shallow merges them. Use `*` to deeply merge.", - document: "a: {thing: {name: Astuff, value: x}, a1: cool}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: {thing: {name: Bstuff, legs: 3}, a1: cool, b1: neat}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\n", - }, - }, - { - description: "Custom types: that are really strings", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse cat\nb: !goat _meow", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse cat_meow\nb: !goat _meow\n", - }, - }, - { - description: "Custom types: that are really numbers", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse 1.2\nb: !goat 2.3", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse 3.5\nb: !goat 2.3\n", - }, - }, - { - skipDoc: true, - document: "a: !horse 2\nb: !goat 2.3", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse 4.3\nb: !goat 2.3\n", - }, - }, - { - skipDoc: true, - document: "a: 2\nb: !goat 2.3", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: 4.3\nb: !goat 2.3\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really ints", - document: "a: !horse 2\nb: !goat 3", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse 5\nb: !goat 3\n", - }, - }, - { - description: "Custom types: that are really arrays", - skipDoc: true, - subdescription: "when custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse [a]\nb: !goat [b]", - expression: `.a += .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse [a, b]\nb: !goat [b]\n", - }, - }, - { - skipDoc: true, - description: "Keep anchors", - document: "a: &horse [1]", - expression: `.a += 2`, - expected: []string{ - "D0, P[], (doc)::a: &horse [1, 2]\n", - }, - }, - { - skipDoc: true, - description: "Add sequence to map", - document: "a: {x: cool}", - expression: `.a += [2]`, - expectedError: "!!seq () cannot be added to a !!map (a)", - }, - { - skipDoc: true, - description: "Add sequence to scalar", - document: "a: cool", - expression: `.a += [2]`, - expectedError: "!!seq () cannot be added to a !!str (a)", - }, -} - -func TestAddOperatorScenarios(t *testing.T) { - for _, tt := range addOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "add", addOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_alternative.go b/external/yq/pkg/yqlib/operator_alternative.go deleted file mode 100755 index ca8569f..0000000 --- a/external/yq/pkg/yqlib/operator_alternative.go +++ /dev/null @@ -1,44 +0,0 @@ -package yqlib - -func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- alternative") - prefs := crossFunctionPreferences{ - CalcWhenEmpty: true, - Calculation: alternativeFunc, - LhsResultValue: func(lhs *CandidateNode) (*CandidateNode, error) { - if lhs == nil { - return nil, nil - } - truthy, err := isTruthy(lhs) - if err != nil { - return nil, err - } - if truthy { - return lhs, nil - } - return nil, nil - }, - } - return crossFunctionWithPrefs(d, context, expressionNode, prefs) -} - -func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - if lhs == nil { - return rhs, nil - } - if rhs == nil { - return lhs, nil - } - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - log.Debugf("Alternative LHS: %v", lhs.Node.Tag) - log.Debugf("- RHS: %v", rhs.Node.Tag) - - isTrue, err := isTruthy(lhs) - if err != nil { - return nil, err - } else if isTrue { - return lhs, nil - } - return rhs, nil -} diff --git a/external/yq/pkg/yqlib/operator_alternative_test.go b/external/yq/pkg/yqlib/operator_alternative_test.go deleted file mode 100755 index 7410663..0000000 --- a/external/yq/pkg/yqlib/operator_alternative_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var alternativeOperatorScenarios = []expressionScenario{ - { - // to match jq - we do not use a readonly clone context on the LHS. - skipDoc: true, - expression: `.b // .c`, - document: `a: bridge`, - expected: []string{ - "D0, P[c], (!!null)::null\n", - }, - }, - { - skipDoc: true, - expression: `(.b // "hello") as $x`, - document: `a: bridge`, - expected: []string{ - "D0, P[], (doc)::a: bridge\n", - }, - }, - { - skipDoc: true, - expression: `.a // .b`, - document: `a: 2`, - expected: []string{ - "D0, P[a], (!!int)::2\n", - }, - }, - { - description: "LHS is defined", - expression: `.a // "hello"`, - document: `{a: bridge}`, - expected: []string{ - "D0, P[a], (!!str)::bridge\n", - }, - }, - { - expression: `select(tag == "seq") // "cat"`, - skipDoc: true, - document: `a: frog`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - description: "LHS is not defined", - expression: `.a // "hello"`, - document: `{}`, - expected: []string{ - "D0, P[], (!!str)::hello\n", - }, - }, - { - description: "LHS is null", - expression: `.a // "hello"`, - document: `{a: ~}`, - expected: []string{ - "D0, P[], (!!str)::hello\n", - }, - }, - { - description: "LHS is false", - expression: `.a // "hello"`, - document: `{a: false}`, - expected: []string{ - "D0, P[], (!!str)::hello\n", - }, - }, - { - description: "RHS is an expression", - expression: `.a // .b`, - document: `{a: false, b: cat}`, - expected: []string{ - "D0, P[b], (!!str)::cat\n", - }, - }, - { - skipDoc: true, - expression: `false // true`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "Update or create - entity exists", - subdescription: "This initialises `a` if it's not present", - expression: "(.a // (.a = 0)) += 1", - document: `a: 1`, - expected: []string{ - "D0, P[], (doc)::a: 2\n", - }, - }, - { - description: "Update or create - entity does not exist", - subdescription: "This initialises `a` if it's not present", - expression: "(.a // (.a = 0)) += 1", - document: `b: camel`, - expected: []string{ - "D0, P[], (!!map)::b: camel\na: 1\n", - }, - }, -} - -func TestAlternativeOperatorScenarios(t *testing.T) { - for _, tt := range alternativeOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "alternative-default-value", alternativeOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_anchors_aliases.go b/external/yq/pkg/yqlib/operator_anchors_aliases.go deleted file mode 100755 index caaf364..0000000 --- a/external/yq/pkg/yqlib/operator_anchors_aliases.go +++ /dev/null @@ -1,300 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("AssignAlias operator!") - - aliasName := "" - if !expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if rhs.MatchingNodes.Front() != nil { - aliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debugf("Setting aliasName : %v", candidate.GetKey()) - - if expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if rhs.MatchingNodes.Front() != nil { - aliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - if aliasName != "" { - candidate.Node.Kind = yaml.AliasNode - candidate.Node.Value = aliasName - } - } - return context, nil -} - -func getAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetAlias operator!") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: "!!str"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - return context.ChildContext(results), nil -} - -func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("AssignAnchor operator!") - - anchorName := "" - if !expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - anchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debugf("Setting anchorName of : %v", candidate.GetKey()) - - if expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - anchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - candidate.Node.Anchor = anchorName - } - return context, nil -} - -func getAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetAnchor operator!") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - anchor := candidate.Node.Anchor - node := &yaml.Node{Kind: yaml.ScalarNode, Value: anchor, Tag: "!!str"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - return context.ChildContext(results), nil -} - -func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- ExplodeOperation") - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - for childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() { - err = explodeNode(childEl.Value.(*CandidateNode).Node, context) - if err != nil { - return Context{}, err - } - } - - } - - return context, nil -} - -func reconstructAliasedMap(node *yaml.Node, context Context) error { - var newContent = list.New() - // can I short cut here by prechecking if there's an anchor in the map? - // no it needs to recurse in overrideEntry. - - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - log.Debugf("traversing %v", keyNode.Value) - if keyNode.Value != "<<" { - err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent)) - if err != nil { - return err - } - } else { - if valueNode.Kind == yaml.SequenceNode { - log.Debugf("an alias merge list!") - for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { - aliasNode := valueNode.Content[index] - err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent)) - if err != nil { - return err - } - } - } else { - log.Debugf("an alias merge!") - err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent)) - if err != nil { - return err - } - } - } - } - node.Content = make([]*yaml.Node, newContent.Len()) - index := 0 - for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() { - node.Content[index] = newEl.Value.(*yaml.Node) - index++ - } - return nil -} - -func explodeNode(node *yaml.Node, context Context) error { - node.Anchor = "" - switch node.Kind { - case yaml.SequenceNode, yaml.DocumentNode: - for index, contentNode := range node.Content { - log.Debugf("exploding index %v", index) - errorInContent := explodeNode(contentNode, context) - if errorInContent != nil { - return errorInContent - } - } - return nil - case yaml.AliasNode: - log.Debugf("its an alias!") - if node.Alias != nil { - node.Kind = node.Alias.Kind - node.Style = node.Alias.Style - node.Tag = node.Alias.Tag - node.Content = deepCloneContent(node.Alias.Content) - node.Value = node.Alias.Value - node.Alias = nil - } - return nil - case yaml.MappingNode: - // //check the map has an alias in it - hasAlias := false - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - if keyNode.Value == "<<" { - hasAlias = true - break - } - } - - if hasAlias { - // this is a slow op, which is why we want to check before running it. - return reconstructAliasedMap(node, context) - } - // this map has no aliases, but it's kids might - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - err := explodeNode(keyNode, context) - if err != nil { - return err - } - err = explodeNode(valueNode, context) - if err != nil { - return err - } - } - return nil - default: - return nil - } -} - -func applyAlias(node *yaml.Node, alias *yaml.Node, aliasIndex int, newContent Context) error { - if alias == nil { - return nil - } - if alias.Kind != yaml.MappingNode { - return fmt.Errorf("merge anchor only supports maps, got %v instead", alias.Tag) - } - for index := 0; index < len(alias.Content); index = index + 2 { - keyNode := alias.Content[index] - log.Debugf("applying alias key %v", keyNode.Value) - valueNode := alias.Content[index+1] - err := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent) - if err != nil { - return err - } - } - return nil -} - -func overrideEntry(node *yaml.Node, key *yaml.Node, value *yaml.Node, startIndex int, newContent Context) error { - - err := explodeNode(value, newContent) - - if err != nil { - return err - } - - for newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() { - valueEl := newEl.Next() // move forward twice - keyNode := newEl.Value.(*yaml.Node) - log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*yaml.Node).Value) - if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil { - log.Debugf("overridign new content") - valueEl.Value = value - return nil - } - newEl = valueEl // move forward twice - } - - for index := startIndex + 2; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - - if keyNode.Value == key.Value && keyNode.Alias == nil { - log.Debugf("content will be overridden at index %v", index) - return nil - } - } - - err = explodeNode(key, newContent) - if err != nil { - return err - } - log.Debugf("adding %v:%v", key.Value, value.Value) - newContent.MatchingNodes.PushBack(key) - newContent.MatchingNodes.PushBack(value) - return nil -} diff --git a/external/yq/pkg/yqlib/operator_anchors_aliases_test.go b/external/yq/pkg/yqlib/operator_anchors_aliases_test.go deleted file mode 100755 index 07a3fdf..0000000 --- a/external/yq/pkg/yqlib/operator_anchors_aliases_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var specDocument = `- &CENTER { x: 1, y: 2 } -- &LEFT { x: 0, y: 2 } -- &BIG { r: 10 } -- &SMALL { r: 1 } -` - -var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n" - -var simpleArrayRef = ` -item_value: &item_value - value: true - -thingOne: - name: item_1 - <<: *item_value - -thingTwo: - name: item_2 - <<: *item_value -` - -var expectedUpdatedArrayRef = `D0, P[], (doc)::item_value: &item_value - value: true -thingOne: - name: item_1 - value: false -thingTwo: - name: item_2 - !!merge <<: *item_value -` - -var anchorOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - description: "merge anchor not map", - document: "a: &a\n - 0\nc:\n <<: [*a]\n", - expectedError: "merge anchor only supports maps, got !!seq instead", - expression: "explode(.)", - }, - { - description: "Merge one map", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : *CENTER\n r: 10\n", - expression: ".[4] | explode(.)", - expected: []string{expectedSpecResult}, - }, - { - description: "Merge multiple maps", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *CENTER, *BIG ]\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, - { - description: "Override", - subdescription: "see https://yaml.org/type/merge.html", - document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", - expression: ".[4] | explode(.)", - expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, - }, - { - description: "Get anchor", - document: `a: &billyBob cat`, - expression: `.a | anchor`, - expected: []string{ - "D0, P[a], (!!str)::billyBob\n", - }, - }, - { - description: "Set anchor", - document: `a: cat`, - expression: `.a anchor = "foobar"`, - expected: []string{ - "D0, P[], (doc)::a: &foobar cat\n", - }, - }, - { - description: "Set anchor relatively using assign-update", - document: `a: {b: cat}`, - expression: `.a anchor |= .b`, - expected: []string{ - "D0, P[], (doc)::a: &cat {b: cat}\n", - }, - }, - { - skipDoc: true, - document: `a: {c: cat}`, - expression: `.a anchor |= .b`, - expected: []string{ - "D0, P[], (doc)::a: {c: cat}\n", - }, - }, - { - skipDoc: true, - document: `a: {c: cat}`, - expression: `.a anchor = .b`, - expected: []string{ - "D0, P[], (doc)::a: {c: cat}\n", - }, - }, - { - description: "Get alias", - document: `{b: &billyBob meow, a: *billyBob}`, - expression: `.a | alias`, - expected: []string{ - "D0, P[a], (!!str)::billyBob\n", - }, - }, - { - description: "Set alias", - document: `{b: &meow purr, a: cat}`, - expression: `.a alias = "meow"`, - expected: []string{ - "D0, P[], (doc)::{b: &meow purr, a: *meow}\n", - }, - }, - { - description: "Set alias to blank does nothing", - document: `{b: &meow purr, a: cat}`, - expression: `.a alias = ""`, - expected: []string{ - "D0, P[], (doc)::{b: &meow purr, a: cat}\n", - }, - }, - { - skipDoc: true, - document: `{b: &meow purr, a: cat}`, - expression: `.a alias = .c`, - expected: []string{ - "D0, P[], (doc)::{b: &meow purr, a: cat}\n", - }, - }, - { - skipDoc: true, - document: `{b: &meow purr, a: cat}`, - expression: `.a alias |= .c`, - expected: []string{ - "D0, P[], (doc)::{b: &meow purr, a: cat}\n", - }, - }, - { - description: "Set alias relatively using assign-update", - document: `{b: &meow purr, a: {f: meow}}`, - expression: `.a alias |= .f`, - expected: []string{ - "D0, P[], (doc)::{b: &meow purr, a: *meow}\n", - }, - }, - { - description: "Explode alias and anchor", - document: `{f : {a: &a cat, b: *a}}`, - expression: `explode(.f)`, - expected: []string{ - "D0, P[], (doc)::{f: {a: cat, b: cat}}\n", - }, - }, - { - description: "Explode with no aliases or anchors", - document: `a: mike`, - expression: `explode(.a)`, - expected: []string{ - "D0, P[], (doc)::a: mike\n", - }, - }, - { - description: "Explode with alias keys", - document: `{f : {a: &a cat, *a: b}}`, - expression: `explode(.f)`, - expected: []string{ - "D0, P[], (doc)::{f: {a: cat, cat: b}}\n", - }, - }, - { - description: "Explode with merge anchors", - document: mergeDocSample, - expression: `explode(.)`, - expected: []string{`D0, P[], (doc)::foo: - a: foo_a - thing: foo_thing - c: foo_c -bar: - b: bar_b - thing: bar_thing - c: bar_c -foobarList: - b: bar_b - thing: foo_thing - c: foobarList_c - a: foo_a -foobar: - c: foo_c - a: foo_a - thing: foobar_thing -`}, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foo* | explode(.) | (. style="flow")`, - expected: []string{ - "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", - "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foo* | explode(explode(.)) | (. style="flow")`, - expected: []string{ - "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", - "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", - "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", - }, - }, - { - skipDoc: true, - document: `{f : {a: &a cat, b: &b {f: *a}, *a: *b}}`, - expression: `explode(.f)`, - expected: []string{ - "D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n", - }, - }, - { - description: "Dereference and update a field", - subdescription: "`Use explode with multiply to dereference an object", - document: simpleArrayRef, - expression: `.thingOne |= explode(.) * {"value": false}`, - expected: []string{expectedUpdatedArrayRef}, - }, -} - -func TestAnchorAliasOperatorScenarios(t *testing.T) { - for _, tt := range anchorOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "anchor-and-alias-operators", anchorOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_array_to_map_test.go b/external/yq/pkg/yqlib/operator_array_to_map_test.go deleted file mode 100755 index 406fec1..0000000 --- a/external/yq/pkg/yqlib/operator_array_to_map_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var arrayToMapScenarios = []expressionScenario{ - { - description: "Simple example", - document: `cool: [null, null, hello]`, - expression: `.cool |= array_to_map`, - expected: []string{ - "D0, P[], (doc)::cool:\n 2: hello\n", - }, - }, -} - -func TestArrayToMapScenarios(t *testing.T) { - for _, tt := range arrayToMapScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "array-to-map", arrayToMapScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_assign.go b/external/yq/pkg/yqlib/operator_assign.go deleted file mode 100755 index f21f452..0000000 --- a/external/yq/pkg/yqlib/operator_assign.go +++ /dev/null @@ -1,101 +0,0 @@ -package yqlib - -type assignPreferences struct { - DontOverWriteAnchor bool - OnlyWriteNull bool - ClobberCustomTags bool -} - -func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation { - return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - rhs.Node = unwrapDoc(rhs.Node) - if !prefs.OnlyWriteNull || lhs.Node.Tag == "!!null" { - lhs.UpdateFrom(rhs, prefs) - } - return lhs, nil - } -} - -// they way *= (multipleAssign) is handled, we set the multiplePrefs -// on the node, not assignPrefs. Long story. -func getAssignPreferences(preferences interface{}) assignPreferences { - prefs := assignPreferences{} - - switch typedPref := preferences.(type) { - case assignPreferences: - prefs = typedPref - case multiplyPreferences: - prefs = typedPref.AssignPrefs - } - return prefs -} - -func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - - prefs := getAssignPreferences(expressionNode.Operation.Preferences) - - log.Debug("assignUpdateOperator prefs: %v", prefs) - - if !expressionNode.Operation.UpdateAssign { - // this works because we already ran against LHS with an editable context. - _, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc(prefs), false) - return context, err - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - - // grab the first value - first := rhs.MatchingNodes.Front() - - if first != nil { - rhsCandidate := first.Value.(*CandidateNode) - rhsCandidate.Node = unwrapDoc(rhsCandidate.Node) - candidate.UpdateFrom(rhsCandidate, prefs) - } - } - - return context, nil -} - -// does not update content or values -func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debug("getting lhs matching nodes for update") - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - - // grab the first value - first := rhs.MatchingNodes.Front() - - if first != nil { - prefs := assignPreferences{} - if expressionNode.Operation.Preferences != nil { - prefs = expressionNode.Operation.Preferences.(assignPreferences) - } - if !prefs.OnlyWriteNull || candidate.Node.Tag == "!!null" { - candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs) - } - } - } - return context, nil -} diff --git a/external/yq/pkg/yqlib/operator_assign_test.go b/external/yq/pkg/yqlib/operator_assign_test.go deleted file mode 100755 index 4a7cd20..0000000 --- a/external/yq/pkg/yqlib/operator_assign_test.go +++ /dev/null @@ -1,235 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var assignOperatorScenarios = []expressionScenario{ - { - description: "Create yaml file", - expression: `.a.b = "cat" | .x = "frog"`, - expected: []string{ - "D0, P[], ()::a:\n b: cat\nx: frog\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: `.a |= .b`, - expected: []string{ - "D0, P[], (doc)::a: null\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: `.a = .b`, - expected: []string{ - "D0, P[], (doc)::a: null\n", - }, - }, - { - skipDoc: true, - description: "self reference", - document: "a: cat", - expression: `.a = [.a]`, - expected: []string{ - "D0, P[], (doc)::a:\n - cat\n", - }, - }, - { - skipDoc: true, - description: "change to number when old value is valid number", - document: `a: "3"`, - expression: `.a = 3`, - expected: []string{ - "D0, P[], (doc)::a: 3\n", - }, - }, - { - skipDoc: true, - description: "change to bool when old value is valid bool", - document: `a: "true"`, - expression: `.a = true`, - expected: []string{ - "D0, P[], (doc)::a: true\n", - }, - }, - { - skipDoc: true, - description: "update custom tag string, dont clobber style", - document: `a: !cat "meow"`, - expression: `.a = "woof"`, - expected: []string{ - "D0, P[], (doc)::a: !cat \"woof\"\n", - }, - }, - { - description: "Update node to be the child value", - document: `{a: {b: {g: foof}}}`, - expression: `.a |= .b`, - expected: []string{ - "D0, P[], (doc)::{a: {g: foof}}\n", - }, - }, - { - description: "Double elements in an array", - document: `[1,2,3]`, - expression: `.[] |= . * 2`, - expected: []string{ - "D0, P[], (doc)::[2, 4, 6]\n", - }, - }, - { - description: "Update node from another file", - subdescription: "Note this will also work when the second file is a scalar (string/number)", - document: `{a: apples}`, - document2: "{b: bob}", - expression: `select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)`, - expected: []string{ - "D0, P[], (doc)::{a: {b: bob}}\n", - }, - }, - { - description: "Update node to be the sibling value", - document: `{a: {b: child}, b: sibling}`, - expression: `.a = .b`, - expected: []string{ - "D0, P[], (doc)::{a: sibling, b: sibling}\n", - }, - }, - { - description: "Updated multiple paths", - document: `{a: fieldA, b: fieldB, c: fieldC}`, - expression: `(.a, .c) = "potato"`, - expected: []string{ - "D0, P[], (doc)::{a: potato, b: fieldB, c: potato}\n", - }, - }, - { - description: "Update string value", - document: `{a: {b: apple}}`, - expression: `.a.b = "frog"`, - expected: []string{ - "D0, P[], (doc)::{a: {b: frog}}\n", - }, - }, - { - description: "Update string value via |=", - subdescription: "Note there is no difference between `=` and `|=` when the RHS is a scalar", - document: `{a: {b: apple}}`, - expression: `.a.b |= "frog"`, - expected: []string{ - "D0, P[], (doc)::{a: {b: frog}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple}}`, - expression: `.a.b | (. |= "frog")`, - expected: []string{ - "D0, P[a b], (!!str)::frog\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple}}`, - expression: `.a.b |= 5`, - expected: []string{ - "D0, P[], (doc)::{a: {b: 5}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple}}`, - expression: `.a.b |= 3.142`, - expected: []string{ - "D0, P[], (doc)::{a: {b: 3.142}}\n", - }, - }, - { - description: "Update deeply selected results", - subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.", - document: `{a: {b: apple, c: cactus}}`, - expression: `(.a[] | select(. == "apple")) = "frog"`, - expected: []string{ - "D0, P[], (doc)::{a: {b: frog, c: cactus}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple, c: cactus}}`, - expression: `(.a.[] | select(. == "apple")) = "frog"`, - expected: []string{ - "D0, P[], (doc)::{a: {b: frog, c: cactus}}\n", - }, - }, - { - description: "Update array values", - document: `[candy, apple, sandy]`, - expression: `(.[] | select(. == "*andy")) = "bogs"`, - expected: []string{ - "D0, P[], (doc)::[bogs, apple, bogs]\n", - }, - }, - { - description: "Update empty object", - dontFormatInputForDoc: true, - document: `{}`, - expression: `.a.b |= "bogs"`, - expected: []string{ - "D0, P[], (doc)::a:\n b: bogs\n", - }, - }, - { - description: "Update node value that has an anchor", - subdescription: "Anchor will remaple", - dontFormatInputForDoc: true, - document: `a: &cool cat`, - expression: `.a = "dog"`, - expected: []string{ - "D0, P[], (doc)::a: &cool dog\n", - }, - }, - { - description: "Update empty object and array", - dontFormatInputForDoc: true, - document: `{}`, - expression: `.a.b.[0] |= "bogs"`, - expected: []string{ - "D0, P[], (doc)::a:\n b:\n - bogs\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `.a.b.[1].c |= "bogs"`, - expected: []string{ - "D0, P[], (doc)::a:\n b:\n - null\n - c: bogs\n", - }, - }, - { - description: "Custom types are maintained by default", - document: "a: !cat meow\nb: !dog woof", - expression: `.a = .b`, - expected: []string{ - "D0, P[], (doc)::a: !cat woof\nb: !dog woof\n", - }, - }, - { - description: "Custom types: clovver", - subdescription: "Use the `c` option to clobber custom tags", - document: "a: !cat meow\nb: !dog woof", - expression: `.a =c .b`, - expected: []string{ - "D0, P[], (doc)::a: !dog woof\nb: !dog woof\n", - }, - }, -} - -func TestAssignOperatorScenarios(t *testing.T) { - for _, tt := range assignOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "assign-update", assignOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_booleans.go b/external/yq/pkg/yqlib/operator_booleans.go deleted file mode 100755 index d919b9a..0000000 --- a/external/yq/pkg/yqlib/operator_booleans.go +++ /dev/null @@ -1,175 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func isTruthyNode(node *yaml.Node) (bool, error) { - value := true - if node.Tag == "!!null" { - return false, nil - } - if node.Kind == yaml.ScalarNode && node.Tag == "!!bool" { - errDecoding := node.Decode(&value) - if errDecoding != nil { - return false, errDecoding - } - - } - return value, nil -} - -func isTruthy(c *CandidateNode) (bool, error) { - node := unwrapDoc(c.Node) - return isTruthyNode(node) -} - -func getBoolean(candidate *CandidateNode) (bool, error) { - if candidate != nil { - candidate.Node = unwrapDoc(candidate.Node) - return isTruthy(candidate) - } - return false, nil -} - -func getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode { - owner := lhs - - if lhs == nil && rhs == nil { - owner = &CandidateNode{} - } else if lhs == nil { - owner = rhs - } - return owner -} - -func returnRhsTruthy(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - owner := getOwner(lhs, rhs) - rhsBool, err := getBoolean(rhs) - if err != nil { - return nil, err - } - - return createBooleanCandidate(owner, rhsBool), nil -} - -func returnLHSWhen(targetBool bool) func(lhs *CandidateNode) (*CandidateNode, error) { - return func(lhs *CandidateNode) (*CandidateNode, error) { - var err error - var lhsBool bool - - if lhsBool, err = getBoolean(lhs); err != nil || lhsBool != targetBool { - return nil, err - } - owner := &CandidateNode{} - if lhs != nil { - owner = lhs - } - return createBooleanCandidate(owner, targetBool), nil - } -} - -func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *yaml.Node) (bool, error) { - for _, node := range sequenceNode.Content { - - if expressionNode != nil { - //need to evaluate the expression against the node - candidate := &CandidateNode{Node: node} - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode) - if err != nil { - return false, err - } - if rhs.MatchingNodes.Len() > 0 { - node = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node - } else { - // no results found, ignore this entry - continue - } - } - - truthy, err := isTruthyNode(node) - if err != nil { - return false, err - } - if truthy == wantBool { - return true, nil - } - } - return false, nil -} - -func allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - if candidateNode.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag) - } - booleanResult, err := findBoolean(false, d, context, expressionNode.RHS, candidateNode) - if err != nil { - return Context{}, err - } - result := createBooleanCandidate(candidate, !booleanResult) - results.PushBack(result) - } - return context.ChildContext(results), nil -} - -func anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - if candidateNode.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag) - } - booleanResult, err := findBoolean(true, d, context, expressionNode.RHS, candidateNode) - if err != nil { - return Context{}, err - } - result := createBooleanCandidate(candidate, booleanResult) - results.PushBack(result) - } - return context.ChildContext(results), nil -} - -func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - prefs := crossFunctionPreferences{ - CalcWhenEmpty: true, - Calculation: returnRhsTruthy, - LhsResultValue: returnLHSWhen(true), - } - return crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs) -} - -func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - prefs := crossFunctionPreferences{ - CalcWhenEmpty: true, - Calculation: returnRhsTruthy, - LhsResultValue: returnLHSWhen(false), - } - return crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs) -} - -func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- notOperation") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debug("notOperation checking %v", candidate) - truthy, errDecoding := isTruthy(candidate) - if errDecoding != nil { - return Context{}, errDecoding - } - result := createBooleanCandidate(candidate, !truthy) - results.PushBack(result) - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_booleans_test.go b/external/yq/pkg/yqlib/operator_booleans_test.go deleted file mode 100755 index f5735bb..0000000 --- a/external/yq/pkg/yqlib/operator_booleans_test.go +++ /dev/null @@ -1,258 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var booleanOperatorScenarios = []expressionScenario{ - { - description: "`or` example", - expression: `true or false`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "b: hi", - expression: `.a or .c`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "b: false", - expression: `.b or .c`, - expected: []string{ - "D0, P[b], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "b: hi", - expression: `select(.a or .b)`, - expected: []string{ - "D0, P[], (doc)::b: hi\n", - }, - }, - { - skipDoc: true, - document: "b: hi", - expression: `select((.a and .b) | not)`, - expected: []string{ - "D0, P[], (doc)::b: hi\n", - }, - }, - { - skipDoc: true, - description: "And should not run 2nd arg if first is false", - expression: `false and test(3)`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - description: "Or should not run 2nd arg if first is true", - expression: `true or test(3)`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "`and` example", - expression: `true and false`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - document: "[{a: bird, b: dog}, {a: frog, b: bird}, {a: cat, b: fly}]", - description: "Matching nodes with select, equals and or", - expression: `[.[] | select(.a == "cat" or .b == "dog")]`, - expected: []string{ - "D0, P[], (!!seq)::- {a: bird, b: dog}\n- {a: cat, b: fly}\n", - }, - }, - { - description: "`any` returns true if any boolean in a given array is true", - document: `[false, true]`, - expression: "any", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "`any` returns false for an empty array", - document: `[]`, - expression: "any", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "`any_c` returns true if any element in the array is true for the given condition.", - document: "a: [rad, awesome]\nb: [meh, whatever]", - expression: `.[] |= any_c(. == "awesome")`, - expected: []string{ - "D0, P[], (doc)::a: true\nb: false\n", - }, - }, - { - skipDoc: true, - document: `[{pet: cat}]`, - expression: `any_c(.name == "harry") as $c`, - expected: []string{ - "D0, P[], (doc)::[{pet: cat}]\n", - }, - }, - { - skipDoc: true, - document: `[{pet: cat}]`, - expression: `all_c(.name == "harry") as $c`, - expected: []string{ - "D0, P[], (doc)::[{pet: cat}]\n", - }, - }, - { - skipDoc: true, - document: `[false, false]`, - expression: "any", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "`all` returns true if all booleans in a given array are true", - document: `[true, true]`, - expression: "all", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `[false, true]`, - expression: "all", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "`all` returns true for an empty array", - document: `[]`, - expression: "all", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "`all_c` returns true if all elements in the array are true for the given condition.", - document: "a: [rad, awesome]\nb: [meh, 12]", - expression: `.[] |= all_c(tag == "!!str")`, - expected: []string{ - "D0, P[], (doc)::a: true\nb: false\n", - }, - }, - { - skipDoc: true, - expression: `false or false`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `{a: true, b: false}`, - expression: `.[] or (false, true)`, - expected: []string{ - "D0, P[a], (!!bool)::true\n", - "D0, P[b], (!!bool)::false\n", - "D0, P[b], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `{a: true, b: false}`, - expression: `.[] and (false, true)`, - expected: []string{ - "D0, P[a], (!!bool)::false\n", - "D0, P[a], (!!bool)::true\n", - "D0, P[b], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `(.a.b or .c) as $x`, - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `(.a.b and .c) as $x`, - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - description: "Not true is false", - expression: `true | not`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Not false is true", - expression: `false | not`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "String values considered to be true", - expression: `"cat" | not`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Empty string value considered to be true", - expression: `"" | not`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Numbers are considered to be true", - expression: `1 | not`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Zero is considered to be true", - expression: `0 | not`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - - description: "Null is considered to be false", - expression: `~ | not`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, -} - -func TestBooleanOperatorScenarios(t *testing.T) { - for _, tt := range booleanOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "boolean-operators", booleanOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_collect.go b/external/yq/pkg/yqlib/operator_collect.go deleted file mode 100755 index a7f2546..0000000 --- a/external/yq/pkg/yqlib/operator_collect.go +++ /dev/null @@ -1,74 +0,0 @@ -package yqlib - -import ( - "container/list" - - yaml "gopkg.in/yaml.v3" -) - -func collectTogether(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*CandidateNode, error) { - collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode) - if err != nil { - return nil, err - } - for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { - resultC := result.Value.(*CandidateNode) - log.Debugf("found this: %v", NodeToString(resultC)) - collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node)) - } - } - return &CandidateNode{Node: collectedNode}, nil -} - -func collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- collectOperation") - - if context.MatchingNodes.Len() == 0 { - node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Value: "[]"} - candidate := &CandidateNode{Node: node} - return context.SingleChildContext(candidate), nil - } - - var evaluateAllTogether = true - for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { - evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether - if !evaluateAllTogether { - break - } - } - - if evaluateAllTogether { - collectedNode, err := collectTogether(d, context, expressionNode.RHS) - if err != nil { - return Context{}, err - } - return context.SingleChildContext(collectedNode), nil - - } - - var results = list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - collectCandidate := candidate.CreateReplacement(collectedNode) - - collectExpResults, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { - resultC := result.Value.(*CandidateNode) - log.Debugf("found this: %v", NodeToString(resultC)) - collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node)) - } - - results.PushBack(collectCandidate) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_collect_object.go b/external/yq/pkg/yqlib/operator_collect_object.go deleted file mode 100755 index 59417a6..0000000 --- a/external/yq/pkg/yqlib/operator_collect_object.go +++ /dev/null @@ -1,101 +0,0 @@ -package yqlib - -import ( - "container/list" - - yaml "gopkg.in/yaml.v3" -) - -/* -[Mike: cat, Bob: dog] -[Thing: rabbit, peter: sam] - -==> cross multiply - -{Mike: cat, Thing: rabbit} -{Mike: cat, peter: sam} -... -*/ - -func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- collectObjectOperation") - - context := originalContext.WritableClone() - - if context.MatchingNodes.Len() == 0 { - node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"} - candidate := &CandidateNode{Node: node} - return context.SingleChildContext(candidate), nil - } - first := context.MatchingNodes.Front().Value.(*CandidateNode) - var rotated = make([]*list.List, len(first.Node.Content)) - - for i := 0; i < len(first.Node.Content); i++ { - rotated[i] = list.New() - } - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidateNode := el.Value.(*CandidateNode) - for i := 0; i < len(first.Node.Content); i++ { - rotated[i].PushBack(candidateNode.CreateChildInArray(i, candidateNode.Node.Content[i])) - } - } - - newObject := list.New() - for i := 0; i < len(first.Node.Content); i++ { - additions, err := collect(d, context.ChildContext(list.New()), rotated[i]) - if err != nil { - return Context{}, err - } - newObject.PushBackList(additions.MatchingNodes) - } - - return context.ChildContext(newObject), nil - -} - -func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) (Context, error) { - if remainingMatches.Len() == 0 { - return context, nil - } - - candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode) - - splatted, err := splat(context.SingleChildContext(candidate), - traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}) - - for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() { - splatEl.Value.(*CandidateNode).Path = nil - } - - if err != nil { - return Context{}, err - } - - if context.MatchingNodes.Len() == 0 { - return collect(d, splatted, remainingMatches) - } - - newAgg := list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - aggCandidate := el.Value.(*CandidateNode) - for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() { - splatCandidate := splatEl.Value.(*CandidateNode) - newCandidate, err := aggCandidate.Copy() - if err != nil { - return Context{}, err - } - - newCandidate.Path = nil - - newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate) - if err != nil { - return Context{}, err - } - newAgg.PushBack(newCandidate) - } - } - return collect(d, context.ChildContext(newAgg), remainingMatches) - -} diff --git a/external/yq/pkg/yqlib/operator_collect_object_test.go b/external/yq/pkg/yqlib/operator_collect_object_test.go deleted file mode 100755 index eda96ad..0000000 --- a/external/yq/pkg/yqlib/operator_collect_object_test.go +++ /dev/null @@ -1,171 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var collectObjectOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `[{name: cat}, {name: dog}]`, - expression: `.[] | {.name: "great"}`, - expected: []string{ - "D0, P[], (!!map)::cat: great\n", - "D0, P[], (!!map)::dog: great\n", - }, - }, - { - skipDoc: true, - document: "a: []", - expression: `.a += [{"key": "att2", "value": "val2"}]`, - expected: []string{ - "D0, P[], (doc)::a:\n - key: att2\n value: val2\n", - }, - }, - { - skipDoc: true, - document: "", - expression: `.a += {"key": "att2", "value": "val2"}`, - expected: []string{ - "D0, P[], ()::a:\n key: att2\n value: val2\n", - }, - }, - { - skipDoc: true, - document: "", - expression: `.a += [0]`, - expected: []string{ - "D0, P[], ()::a:\n - 0\n", - }, - }, - { - description: `Collect empty object`, - document: ``, - expression: `{}`, - expected: []string{ - "D0, P[], (!!map)::{}\n", - }, - }, - { - description: `Wrap (prefix) existing object`, - document: "{name: Mike}\n", - expression: `{"wrap": .}`, - expected: []string{ - "D0, P[], (!!map)::wrap: {name: Mike}\n", - }, - }, - { - skipDoc: true, - document: "{name: Mike}\n", - document2: "{name: Bob}\n", - expression: `{"wrap": .}`, - expected: []string{ - "D0, P[], (!!map)::wrap: {name: Mike}\n", - "D0, P[], (!!map)::wrap: {name: Bob}\n", - }, - }, - { - skipDoc: true, - document: "{name: Mike}\n---\n{name: Bob}", - expression: `{"wrap": .}`, - expected: []string{ - "D0, P[], (!!map)::wrap: {name: Mike}\n", - "D0, P[], (!!map)::wrap: {name: Bob}\n", - }, - }, - { - skipDoc: true, - document: `{name: Mike, age: 32}`, - expression: `{.name: .age}`, - expected: []string{ - "D0, P[], (!!map)::Mike: 32\n", - }, - }, - { - description: `Using splat to create multiple objects`, - document: `{name: Mike, pets: [cat, dog]}`, - expression: `{.name: .pets.[]}`, - expected: []string{ - "D0, P[], (!!map)::Mike: cat\n", - "D0, P[], (!!map)::Mike: dog\n", - }, - }, - { - description: `Working with multiple documents`, - dontFormatInputForDoc: false, - document: "{name: Mike, pets: [cat, dog]}\n---\n{name: Rosey, pets: [monkey, sheep]}", - expression: `{.name: .pets.[]}`, - expected: []string{ - "D0, P[], (!!map)::Mike: cat\n", - "D0, P[], (!!map)::Mike: dog\n", - "D0, P[], (!!map)::Rosey: monkey\n", - "D0, P[], (!!map)::Rosey: sheep\n", - }, - }, - { - skipDoc: true, - document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, - expression: `{.name: .pets.[], "f":.food.[]}`, - expected: []string{ - "D0, P[], (!!map)::Mike: cat\nf: hotdog\n", - "D0, P[], (!!map)::Mike: cat\nf: burger\n", - "D0, P[], (!!map)::Mike: dog\nf: hotdog\n", - "D0, P[], (!!map)::Mike: dog\nf: burger\n", - }, - }, - { - skipDoc: true, - document: "name: Mike\npets:\n cows:\n - apl\n - bba", - document2: "name: Rosey\npets:\n sheep:\n - frog\n - meow", - expression: `{"a":.name, "b":.pets}`, - expected: []string{ - "D0, P[], (!!map)::a: Mike\nb:\n cows:\n - apl\n - bba\n", - "D0, P[], (!!map)::a: Rosey\nb:\n sheep:\n - frog\n - meow\n", - }, - }, - { - description: "Creating yaml from scratch", - document: ``, - expression: `{"wrap": "frog"}`, - expected: []string{ - "D0, P[], (!!map)::wrap: frog\n", - }, - }, - { - skipDoc: true, - expression: `{"wrap": "frog", "bing": "bong"}`, - expected: []string{ - "D0, P[], (!!map)::wrap: frog\nbing: bong\n", - }, - }, - { - skipDoc: true, - document: `{name: Mike}`, - expression: `{"wrap": .}`, - expected: []string{ - "D0, P[], (!!map)::wrap: {name: Mike}\n", - }, - }, - { - skipDoc: true, - document: `{name: Mike}`, - expression: `{"wrap": {"further": .}} | (.. style= "flow")`, - expected: []string{ - "D0, P[], (!!map)::{wrap: {further: {name: Mike}}}\n", - }, - }, - { - description: "Creating yaml from scratch with multiple objects", - expression: `(.a.b = "foo") | (.d.e = "bar")`, - expected: []string{ - "D0, P[], ()::a:\n b: foo\nd:\n e: bar\n", - }, - }, -} - -func TestCollectObjectOperatorScenarios(t *testing.T) { - for _, tt := range collectObjectOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "create-collect-into-object", collectObjectOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_collect_test.go b/external/yq/pkg/yqlib/operator_collect_test.go deleted file mode 100755 index b75f95f..0000000 --- a/external/yq/pkg/yqlib/operator_collect_test.go +++ /dev/null @@ -1,129 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var collectOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: ``, - expression: `.a += [0]`, - expected: []string{ - "D0, P[], ()::a:\n - 0\n", - }, - }, - { - skipDoc: true, - expression: `[1,2,3]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", - }, - }, - { - skipDoc: true, - description: "update in collect", - expression: `[.a = "cat"]`, - expected: []string{ - "D0, P[], (!!seq)::- a: cat\n", - }, - }, - { - description: "Collect empty", - document: ``, - expression: `[]`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - skipDoc: true, - document: "{a: apple}\n---\n{b: frog}", - expression: `[.]`, - expected: []string{ - "D0, P[], (!!seq)::- {a: apple}\n- {b: frog}\n", - }, - }, - { - description: "with comments", - skipDoc: true, - document: "# abc\n[{a: apple}]\n\n# xyz\n", - - expression: `[.[]]`, - expected: []string{ - "D0, P[], (!!seq)::- {a: apple}\n", - }, - }, - { - skipDoc: true, - document: ``, - expression: `[3]`, - expected: []string{ - "D0, P[], (!!seq)::- 3\n", - }, - }, - { - description: "Collect single", - document: ``, - expression: `["cat"]`, - expected: []string{ - "D0, P[], (!!seq)::- cat\n", - }, - }, - { - document: ``, - skipDoc: true, - expression: `[true]`, - expected: []string{ - "D0, P[], (!!seq)::- true\n", - }, - }, - { - description: "Collect many", - document: `{a: cat, b: dog}`, - expression: `[.a, .b]`, - expected: []string{ - "D0, P[], (!!seq)::- cat\n- dog\n", - }, - }, - { - document: ``, - skipDoc: true, - expression: `collect(1)`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n", - }, - }, - { - document: `[1,2,3]`, - skipDoc: true, - expression: `[.[]]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", - }, - }, - { - document: `a: {b: [1,2,3]}`, - expression: `[.a.b.[]]`, - skipDoc: true, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", - }, - }, - { - skipDoc: true, - document: `[{name: cat, thing: bor}, {name: dog}]`, - expression: `.[] | [.name]`, - expected: []string{ - "D0, P[0], (!!seq)::- cat\n", - "D0, P[1], (!!seq)::- dog\n", - }, - }, -} - -func TestCollectOperatorScenarios(t *testing.T) { - for _, tt := range collectOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "collect-into-array", collectOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_column.go b/external/yq/pkg/yqlib/operator_column.go deleted file mode 100755 index ac7d339..0000000 --- a/external/yq/pkg/yqlib/operator_column.go +++ /dev/null @@ -1,23 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func columnOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("columnOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.Node.Column), Tag: "!!int"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_column_test.go b/external/yq/pkg/yqlib/operator_column_test.go deleted file mode 100755 index 3684588..0000000 --- a/external/yq/pkg/yqlib/operator_column_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var columnOperatorScenarios = []expressionScenario{ - { - description: "Returns column of _value_ node", - document: "a: cat\nb: bob", - expression: `.b | column`, - expected: []string{ - "D0, P[b], (!!int)::4\n", - }, - }, - { - description: "Returns column of _key_ node", - subdescription: "Pipe through the key operator to get the column of the key", - document: "a: cat\nb: bob", - expression: `.b | key | column`, - expected: []string{ - "D0, P[b], (!!int)::1\n", - }, - }, - { - description: "First column is 1", - document: "a: cat", - expression: `.a | key | column`, - expected: []string{ - "D0, P[a], (!!int)::1\n", - }, - }, - { - description: "No column data is 0", - expression: `{"a": "new entry"} | column`, - expected: []string{ - "D0, P[], (!!int)::0\n", - }, - }, -} - -func TestColumnOperatorScenarios(t *testing.T) { - for _, tt := range columnOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "column", columnOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_comments.go b/external/yq/pkg/yqlib/operator_comments.go deleted file mode 100755 index 5bbe2e5..0000000 --- a/external/yq/pkg/yqlib/operator_comments.go +++ /dev/null @@ -1,124 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "container/list" - "regexp" - - yaml "gopkg.in/yaml.v3" -) - -type commentOpPreferences struct { - LineComment bool - HeadComment bool - FootComment bool -} - -func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("AssignComments operator!") - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - - preferences := expressionNode.Operation.Preferences.(commentOpPreferences) - - comment := "" - if !expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - comment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - if expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - comment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - log.Debugf("Setting comment of : %v", candidate.GetKey()) - if preferences.LineComment { - candidate.Node.LineComment = comment - } - if preferences.HeadComment { - candidate.Node.HeadComment = comment - candidate.LeadingContent = "" // clobber the leading content, if there was any. - } - if preferences.FootComment && candidate.Node.Kind == yaml.DocumentNode && comment != "" { - candidate.TrailingContent = "# " + comment - } else if preferences.FootComment && candidate.Node.Kind == yaml.DocumentNode { - candidate.TrailingContent = comment - - } else if preferences.FootComment && candidate.Node.Kind != yaml.DocumentNode { - candidate.Node.FootComment = comment - candidate.TrailingContent = "" - } - - } - return context, nil -} - -func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - preferences := expressionNode.Operation.Preferences.(commentOpPreferences) - var startCommentCharaterRegExp = regexp.MustCompile(`^# `) - var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `) - - log.Debugf("GetComments operator!") - var results = list.New() - - yamlPrefs := NewDefaultYamlPreferences() - yamlPrefs.PrintDocSeparators = false - yamlPrefs.UnwrapScalar = false - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - comment := "" - if preferences.LineComment { - comment = candidate.Node.LineComment - } else if preferences.HeadComment && candidate.LeadingContent != "" { - var chompRegexp = regexp.MustCompile(`\n$`) - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - var encoder = NewYamlEncoder(2, false, yamlPrefs) - if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil { - return Context{}, err - } - if err := writer.Flush(); err != nil { - return Context{}, err - } - comment = output.String() - comment = chompRegexp.ReplaceAllString(comment, "") - } else if preferences.HeadComment { - comment = candidate.Node.HeadComment - } else if preferences.FootComment && candidate.Node.Kind == yaml.DocumentNode && candidate.TrailingContent != "" { - comment = candidate.TrailingContent - } else if preferences.FootComment { - comment = candidate.Node.FootComment - } - comment = startCommentCharaterRegExp.ReplaceAllString(comment, "") - comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n") - - node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"} - result := candidate.CreateReplacement(node) - result.LeadingContent = "" // don't include the leading yaml content when retrieving a comment - results.PushBack(result) - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_comments_test.go b/external/yq/pkg/yqlib/operator_comments_test.go deleted file mode 100755 index e117e18..0000000 --- a/external/yq/pkg/yqlib/operator_comments_test.go +++ /dev/null @@ -1,270 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var expectedWhereIsMyCommentMapKey = `D0, P[], (!!seq)::- p: "" - isKey: false - hc: "" - lc: "" - fc: "" -- p: hello - isKey: true - hc: "" - lc: hello-world-comment - fc: "" -- p: hello - isKey: false - hc: "" - lc: "" - fc: "" -- p: hello.message - isKey: true - hc: "" - lc: "" - fc: "" -- p: hello.message - isKey: false - hc: "" - lc: "" - fc: "" -` - -var expectedWhereIsMyCommentArray = `D0, P[], (!!seq)::- p: "" - isKey: false - hc: "" - lc: "" - fc: "" -- p: name - isKey: true - hc: "" - lc: "" - fc: "" -- p: name - isKey: false - hc: "" - lc: "" - fc: "" -- p: name.0 - isKey: false - hc: under-name-comment - lc: "" - fc: "" -` - -var commentOperatorScenarios = []expressionScenario{ - { - description: "Set line comment", - subdescription: "Set the comment on the key node for more reliability (see below).", - document: `a: cat`, - expression: `.a line_comment="single"`, - expected: []string{ - "D0, P[], (doc)::a: cat # single\n", - }, - }, - { - description: "Set line comment of a maps/arrays", - subdescription: "For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.", - document: "a:\n b: things", - expression: `(.a | key) line_comment="single"`, - expected: []string{ - "D0, P[], (doc)::a: # single\n b: things\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: dog", - expression: `.a line_comment=.b`, - expected: []string{ - "D0, P[], (doc)::a: cat # dog\nb: dog\n", - }, - }, - { - skipDoc: true, - document: "a: cat\n---\na: dog", - expression: `.a line_comment |= documentIndex`, - expected: []string{ - "D0, P[], (doc)::a: cat # 0\n", - "D1, P[], (doc)::a: dog # 1\n", - }, - }, - { - description: "Use update assign to perform relative updates", - document: "a: cat\nb: dog", - expression: `.. line_comment |= .`, - expected: []string{ - "D0, P[], (!!map)::a: cat # cat\nb: dog # dog\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: dog", - expression: `.. comments |= .`, - expected: []string{ - "D0, P[], (!!map)::a: cat # cat\n# cat\n\n# cat\nb: dog # dog\n# dog\n\n# dog\n", - }, - }, - { - description: "Where is the comment - map key example", - subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'hello-world-comment' is actually on the 'hello' key", - document: "hello: # hello-world-comment\n message: world", - expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, - expected: []string{ - expectedWhereIsMyCommentMapKey, - }, - }, - { - description: "Retrieve comment - map key example", - subdescription: "From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment", - document: "hello: # hello-world-comment\n message: world", - expression: `.hello | key | line_comment`, - expected: []string{ - "D0, P[hello], (!!str)::hello-world-comment\n", - }, - }, - { - description: "Where is the comment - array example", - subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'under-name-comment' is actually on the first child", - document: "name:\n # under-name-comment\n - first-array-child", - expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, - expected: []string{ - expectedWhereIsMyCommentArray, - }, - }, - { - description: "Retrieve comment - array example", - subdescription: "From the previous example, we know that the comment is on the first child as a headComment", - document: "name:\n # under-name-comment\n - first-array-child", - expression: `.name[0] | headComment`, - expected: []string{ - "D0, P[name 0], (!!str)::under-name-comment\n", - }, - }, - { - description: "Set head comment", - document: `a: cat`, - expression: `. head_comment="single"`, - expected: []string{ - "D0, P[], (doc)::# single\n\na: cat\n", - }, - }, - { - description: "Set head comment of a map entry", - document: "f: foo\na:\n b: cat", - expression: `(.a | key) head_comment="single"`, - expected: []string{ - "D0, P[], (doc)::f: foo\n# single\na:\n b: cat\n", - }, - }, - { - description: "Set foot comment, using an expression", - document: `a: cat`, - expression: `. foot_comment=.a`, - expected: []string{ - "D0, P[], (doc)::a: cat\n# cat\n", - }, - }, - { - skipDoc: true, - description: "Set foot comment, using an expression", - document: "a: cat\n\n# hi", - expression: `. foot_comment=""`, - expected: []string{ - "D0, P[], (doc)::a: cat\n", - }, - }, - { - skipDoc: true, - document: `a: cat`, - expression: `. foot_comment=.b.d`, - expected: []string{ - "D0, P[], (doc)::a: cat\n", - }, - }, - { - skipDoc: true, - document: `a: cat`, - expression: `. foot_comment|=.b.d`, - expected: []string{ - "D0, P[], (doc)::a: cat\n", - }, - }, - { - description: "Remove comment", - document: "a: cat # comment\nb: dog # leave this", - expression: `.a line_comment=""`, - expected: []string{ - "D0, P[], (doc)::a: cat\nb: dog # leave this\n", - }, - }, - { - description: "Remove (strip) all comments", - subdescription: "Note the use of `...` to ensure key nodes are included.", - document: "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment", - expression: `... comments=""`, - expected: []string{ - "D0, P[], (!!map)::a: cat\nb:\n", - }, - }, - { - description: "Get line comment", - document: "# welcome!\n\na: cat # meow\n\n# have a great day", - expression: `.a | line_comment`, - expected: []string{ - "D0, P[a], (!!str)::meow\n", - }, - }, - { - description: "Get head comment", - dontFormatInputForDoc: true, - document: "# welcome!\n\na: cat # meow\n\n# have a great day", - expression: `. | head_comment`, - expected: []string{ - "D0, P[], (!!str)::welcome!\n", - }, - }, - { - skipDoc: true, - description: "strip trailing comment recurse all", - document: "a: cat\n\n# haha", - expression: `... comments= ""`, - expected: []string{ - "D0, P[], (!!map)::a: cat\n", - }, - }, - { - skipDoc: true, - description: "strip trailing comment recurse values", - document: "a: cat\n\n# haha", - expression: `.. comments= ""`, - expected: []string{ - "D0, P[], (!!map)::a: cat\n", - }, - }, - { - description: "Head comment with document split", - dontFormatInputForDoc: true, - document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day", - expression: `head_comment`, - expected: []string{ - "D0, P[], (!!str)::welcome!\nbob\n", - }, - }, - { - description: "Get foot comment", - dontFormatInputForDoc: true, - document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really", - expression: `. | foot_comment`, - expected: []string{ - "D0, P[], (!!str)::have a great day\nno really\n", - }, - }, -} - -func TestCommentOperatorScenarios(t *testing.T) { - for _, tt := range commentOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "comment-operators", commentOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_contains.go b/external/yq/pkg/yqlib/operator_contains.go deleted file mode 100755 index d07a645..0000000 --- a/external/yq/pkg/yqlib/operator_contains.go +++ /dev/null @@ -1,111 +0,0 @@ -package yqlib - -import ( - "fmt" - "strings" - - yaml "gopkg.in/yaml.v3" -) - -func containsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - return crossFunction(d, context.ReadOnlyClone(), expressionNode, containsWithNodes, false) -} - -func containsArrayElement(array *yaml.Node, item *yaml.Node) (bool, error) { - for index := 0; index < len(array.Content); index = index + 1 { - containedInArray, err := contains(array.Content[index], item) - if err != nil { - return false, err - } - if containedInArray { - return true, nil - } - } - return false, nil -} - -func containsArray(lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - if rhs.Kind != yaml.SequenceNode { - return containsArrayElement(lhs, rhs) - } - for index := 0; index < len(rhs.Content); index = index + 1 { - itemInArray, err := containsArrayElement(lhs, rhs.Content[index]) - if err != nil { - return false, err - } - if !itemInArray { - return false, nil - } - } - return true, nil -} - -func containsObject(lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - if rhs.Kind != yaml.MappingNode { - return false, nil - } - for index := 0; index < len(rhs.Content); index = index + 2 { - rhsKey := rhs.Content[index] - rhsValue := rhs.Content[index+1] - log.Debugf("Looking for %v in the lhs", rhsKey.Value) - lhsKeyIndex := findInArray(lhs, rhsKey) - log.Debugf("index is %v", lhsKeyIndex) - if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 { - return false, nil - } - lhsValue := lhs.Content[lhsKeyIndex+1] - log.Debugf("lhsValue is %v", lhsValue.Value) - - itemInArray, err := contains(lhsValue, rhsValue) - log.Debugf("rhsValue is %v", rhsValue.Value) - if err != nil { - return false, err - } - if !itemInArray { - return false, nil - } - } - return true, nil -} - -func containsScalars(lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - if lhs.Tag == "!!str" { - return strings.Contains(lhs.Value, rhs.Value), nil - } - return lhs.Value == rhs.Value, nil -} - -func contains(lhs *yaml.Node, rhs *yaml.Node) (bool, error) { - switch lhs.Kind { - case yaml.MappingNode: - return containsObject(lhs, rhs) - case yaml.SequenceNode: - return containsArray(lhs, rhs) - case yaml.ScalarNode: - if rhs.Kind != yaml.ScalarNode || lhs.Tag != rhs.Tag { - return false, nil - } - if lhs.Tag == "!!null" { - return rhs.Tag == "!!null", nil - } - return containsScalars(lhs, rhs) - } - - return false, fmt.Errorf("%v not yet supported for contains", lhs.Tag) -} - -func containsWithNodes(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - - if lhs.Node.Kind != rhs.Node.Kind { - return nil, fmt.Errorf("%v cannot check contained in %v", rhs.Node.Tag, lhs.Node.Tag) - } - - result, err := contains(lhs.Node, rhs.Node) - if err != nil { - return nil, err - } - - return createBooleanCandidate(lhs, result), nil -} diff --git a/external/yq/pkg/yqlib/operator_contains_test.go b/external/yq/pkg/yqlib/operator_contains_test.go deleted file mode 100755 index f714ace..0000000 --- a/external/yq/pkg/yqlib/operator_contains_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package yqlib - -import "testing" - -var containsOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - expression: `null | contains(~)`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - expression: `3 | contains(3)`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - expression: `3 | contains(32)`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Array contains array", - subdescription: "Array is equal or subset of", - document: `["foobar", "foobaz", "blarp"]`, - expression: `contains(["baz", "bar"])`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "Array has a subset array", - subdescription: "Subtract the superset array from the subset, if there's anything left, it's not a subset", - document: `["foobar", "foobaz", "blarp"]`, - expression: `["baz", "bar"] - . | length == 0`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - expression: `["dog", "cat", "giraffe"] | contains(["camel"])`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Object included in array", - document: `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`, - expression: `contains({"bar": [{"barp": 12}]})`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "Object not included in array", - document: `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`, - expression: `contains({"foo": 12, "bar": [{"barp": 15}]})`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "String contains substring", - document: `"foobar"`, - expression: `contains("bar")`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "String equals string", - document: `"meow"`, - expression: `contains("meow")`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, -} - -func TestContainsOperatorScenarios(t *testing.T) { - for _, tt := range containsOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "contains", containsOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_create_map.go b/external/yq/pkg/yqlib/operator_create_map.go deleted file mode 100755 index 84582c9..0000000 --- a/external/yq/pkg/yqlib/operator_create_map.go +++ /dev/null @@ -1,85 +0,0 @@ -package yqlib - -import ( - "container/list" - - "gopkg.in/yaml.v3" -) - -func createMapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- createMapOperation") - - //each matchingNodes entry should turn into a sequence of keys to create. - //then collect object should do a cross function of the same index sequence for all matches. - - var path []interface{} - - var document uint - - sequences := list.New() - - if context.MatchingNodes.Len() > 0 { - - for matchingNodeEl := context.MatchingNodes.Front(); matchingNodeEl != nil; matchingNodeEl = matchingNodeEl.Next() { - matchingNode := matchingNodeEl.Value.(*CandidateNode) - sequenceNode, err := sequenceFor(d, context, matchingNode, expressionNode) - if err != nil { - return Context{}, err - } - sequences.PushBack(sequenceNode) - } - } else { - sequenceNode, err := sequenceFor(d, context, nil, expressionNode) - if err != nil { - return Context{}, err - } - sequences.PushBack(sequenceNode) - } - - return context.SingleChildContext(&CandidateNode{Node: listToNodeSeq(sequences), Document: document, Path: path}), nil - -} - -func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateNode, expressionNode *ExpressionNode) (*CandidateNode, error) { - var path []interface{} - var document uint - var matches = list.New() - - if matchingNode != nil { - path = matchingNode.Path - document = matchingNode.Document - matches.PushBack(matchingNode) - } - - mapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode, - func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - node := yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - log.Debugf("LHS:", NodeToString(lhs)) - log.Debugf("RHS:", NodeToString(rhs)) - node.Content = []*yaml.Node{ - unwrapDoc(lhs.Node), - unwrapDoc(rhs.Node), - } - - return &CandidateNode{Node: &node, Document: document, Path: path}, nil - }, false) - - if err != nil { - return nil, err - } - innerList := listToNodeSeq(mapPairs.MatchingNodes) - innerList.Style = yaml.FlowStyle - return &CandidateNode{Node: innerList, Document: document, Path: path}, nil -} - -// NOTE: here the document index gets dropped so we -// no longer know where the node originates from. -func listToNodeSeq(list *list.List) *yaml.Node { - node := yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for entry := list.Front(); entry != nil; entry = entry.Next() { - entryCandidate := entry.Value.(*CandidateNode) - log.Debugf("Collecting %v into sequence", NodeToString(entryCandidate)) - node.Content = append(node.Content, entryCandidate.Node) - } - return &node -} diff --git a/external/yq/pkg/yqlib/operator_create_map_test.go b/external/yq/pkg/yqlib/operator_create_map_test.go deleted file mode 100755 index 6a97246..0000000 --- a/external/yq/pkg/yqlib/operator_create_map_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var createMapOperatorScenarios = []expressionScenario{ - { - document: ``, - expression: `"frog": "jumps"`, - expected: []string{ - "D0, P[], (!!seq)::- [{frog: jumps}]\n", - }, - }, - { - document: `{name: Mike, age: 32}`, - expression: `.name: .age`, - expected: []string{ - "D0, P[], (!!seq)::- [{Mike: 32}]\n", - }, - }, - { - document: `{name: Mike, pets: [cat, dog]}`, - expression: `.name: .pets.[]`, - expected: []string{ - "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", - }, - }, - { - document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, - expression: `.name: .pets.[], "f":.food.[]`, - expected: []string{ - "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", - "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n", - }, - }, - { - document: "{name: Mike, pets: [cat, dog], food: [hotdog, burger]}\n---\n{name: Fred, pets: [mouse], food: [pizza, onion, apple]}", - expression: `.name: .pets.[], "f":.food.[]`, - expected: []string{ - "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n- [{Fred: mouse}]\n", - "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n- [{f: pizza}, {f: onion}, {f: apple}]\n", - }, - }, - { - document: `{name: Mike, pets: {cows: [apl, bba]}}`, - expression: `"a":.name, "b":.pets`, - expected: []string{ - "D0, P[], (!!seq)::- [{a: Mike}]\n", - "D0, P[], (!!seq)::- [{b: {cows: [apl, bba]}}]\n", - }, - }, - { - document: `{name: Mike}`, - expression: `"wrap": .`, - expected: []string{ - "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n", - }, - }, - { - document: "{name: Mike}\n---\n{name: Bob}", - expression: `"wrap": .`, - expected: []string{ - "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", - }, - }, - { - document: "{name: Mike}\n---\n{name: Bob}", - expression: `"wrap": ., .name: "great"`, - expected: []string{ - "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", - "D0, P[], (!!seq)::- [{Mike: great}]\n- [{Bob: great}]\n", - }, - }, -} - -func TestCreateMapOperatorScenarios(t *testing.T) { - for _, tt := range createMapOperatorScenarios { - testScenario(t, &tt) - } -} diff --git a/external/yq/pkg/yqlib/operator_datetime.go b/external/yq/pkg/yqlib/operator_datetime.go deleted file mode 100755 index c3ec05f..0000000 --- a/external/yq/pkg/yqlib/operator_datetime.go +++ /dev/null @@ -1,131 +0,0 @@ -package yqlib - -import ( - "container/list" - "errors" - "fmt" - "time" - - "gopkg.in/yaml.v3" -) - -func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) { - result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode) - - if err != nil { - return "", err - } else if result.MatchingNodes.Len() == 0 { - return "", fmt.Errorf("could not find %v for format_time", parameterName) - } - - return result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value, nil -} - -func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType { - layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS) - if err != nil { - return Context{}, fmt.Errorf("could not get date time format: %w", err) - } - context.SetDateTimeLayout(layout) - return d.GetMatchingNodes(context, expressionNode.RHS.RHS) - - } - return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; )`) - -} - -// for unit tests -var Now = time.Now - -func nowOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - node := &yaml.Node{ - Tag: "!!timestamp", - Kind: yaml.ScalarNode, - Value: Now().Format(time.RFC3339), - } - - return context.SingleChildContext(&CandidateNode{Node: node}), nil - -} - -func parseDateTime(layout string, datestring string) (time.Time, error) { - - parsedTime, err := time.Parse(layout, datestring) - if err != nil && layout == time.RFC3339 { - // try parsing the date time with only the date - return time.Parse("2006-01-02", datestring) - } - return parsedTime, err - -} - -func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - format, err := getStringParamter("format", d, context, expressionNode.RHS) - layout := context.GetDateTimeLayout() - - if err != nil { - return Context{}, err - } - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - parsedTime, err := parseDateTime(layout, candidate.Node.Value) - if err != nil { - return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err) - } - formattedTimeStr := parsedTime.Format(format) - - node, errorReading := parseSnippet(formattedTimeStr) - if errorReading != nil { - log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading) - node = &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: formattedTimeStr, - } - } - - results.PushBack(candidate.CreateReplacement(node)) - } - - return context.ChildContext(results), nil -} - -func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS) - layout := context.GetDateTimeLayout() - - if err != nil { - return Context{}, err - } - var results = list.New() - - timezone, err := time.LoadLocation(timezoneStr) - if err != nil { - return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err) - } - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - parsedTime, err := parseDateTime(layout, candidate.Node.Value) - if err != nil { - return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err) - } - tzTime := parsedTime.In(timezone) - - node := &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: candidate.Node.Tag, - Value: tzTime.Format(layout), - } - - results.PushBack(candidate.CreateReplacement(node)) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_datetime_test.go b/external/yq/pkg/yqlib/operator_datetime_test.go deleted file mode 100755 index a1d2692..0000000 --- a/external/yq/pkg/yqlib/operator_datetime_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var dateTimeOperatorScenarios = []expressionScenario{ - { - description: "Format: from standard RFC3339 format", - subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.", - document: `a: 2001-12-15T02:59:43.1Z`, - expression: `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM\n", - }, - }, - { - description: "Format: from custom date time", - subdescription: "Use with_dtf to set a custom datetime format for parsing.", - document: `a: Saturday, 15-Dec-01 at 2:59AM`, - expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`, - expected: []string{ - "D0, P[], (doc)::a: 2001-12-15\n", - }, - }, - { - description: "Format: get the day of the week", - document: `a: 2001-12-15`, - expression: `.a | format_datetime("Monday")`, - expected: []string{ - "D0, P[a], (!!str)::Saturday\n", - }, - }, - { - description: "Now", - document: "a: cool", - expression: `.updated = now`, - expected: []string{ - "D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n", - }, - }, - { - description: "Timezone: from standard RFC3339 format", - subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.", - - document: "a: cool", - expression: `.updated = (now | tz("Australia/Sydney"))`, - expected: []string{ - "D0, P[], (doc)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n", - }, - }, - { - description: "Timezone: with custom format", - subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", - document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", - }, - }, - { - description: "Add and tz custom format", - subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", - document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", - }, - }, - { - description: "Date addition", - document: `a: 2021-01-01T00:00:00Z`, - expression: `.a += "3h10m"`, - expected: []string{ - "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", - }, - }, - { - description: "Date subtraction", - subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.", - document: `a: 2021-01-01T03:10:00Z`, - expression: `.a -= "3h10m"`, - expected: []string{ - "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", - }, - }, - { - description: "Date addition - custom format", - document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", - }, - }, - { - description: "Date script with custom format", - subdescription: "You can embed full expressions in with_dtf if needed.", - document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n", - }, - }, - { - description: "allow comma", - skipDoc: true, - document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", - expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", - }, - }, -} - -func TestDatetimeOperatorScenarios(t *testing.T) { - for _, tt := range dateTimeOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_delete.go b/external/yq/pkg/yqlib/operator_delete.go deleted file mode 100755 index eb475da..0000000 --- a/external/yq/pkg/yqlib/operator_delete.go +++ /dev/null @@ -1,92 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - //need to iterate backwards to ensure correct indices when deleting multiple - for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() { - candidate := el.Value.(*CandidateNode) - - if candidate.Node.Kind == yaml.DocumentNode { - //need to delete this node from context. - newResults := list.New() - for item := context.MatchingNodes.Front(); item != nil; item = item.Next() { - nodeInContext := item.Value.(*CandidateNode) - if nodeInContext.Node != candidate.Node { - newResults.PushBack(nodeInContext) - } else { - log.Info("Need to delete this %v", NodeToString(nodeInContext)) - } - } - return context.ChildContext(newResults), nil - } else if candidate.Parent == nil { - //problem: context may already be '.a' and then I pass in '.a.a2'. - // should pass in .a2. - log.Info("Could not find parent of %v", NodeToString(candidate)) - return context, nil - } - - parentNode := candidate.Parent.Node - childPath := candidate.Path[len(candidate.Path)-1] - - if parentNode.Kind == yaml.MappingNode { - deleteFromMap(candidate.Parent, childPath) - } else if parentNode.Kind == yaml.SequenceNode { - deleteFromArray(candidate.Parent, childPath) - } else { - return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag) - } - } - return context, nil -} - -func deleteFromMap(candidate *CandidateNode, childPath interface{}) { - log.Debug("deleteFromMap") - node := unwrapDoc(candidate.Node) - contents := node.Content - newContents := make([]*yaml.Node, 0) - - for index := 0; index < len(contents); index = index + 2 { - key := contents[index] - value := contents[index+1] - - childCandidate := candidate.CreateChildInMap(key, value) - - shouldDelete := key.Value == childPath - - log.Debugf("shouldDelete %v ? %v", childCandidate.GetKey(), shouldDelete) - - if !shouldDelete { - newContents = append(newContents, key, value) - } - } - node.Content = newContents -} - -func deleteFromArray(candidate *CandidateNode, childPath interface{}) { - log.Debug("deleteFromArray") - node := unwrapDoc(candidate.Node) - contents := node.Content - newContents := make([]*yaml.Node, 0) - - for index := 0; index < len(contents); index = index + 1 { - value := contents[index] - - shouldDelete := fmt.Sprintf("%v", index) == fmt.Sprintf("%v", childPath) - - if !shouldDelete { - newContents = append(newContents, value) - } - } - node.Content = newContents -} diff --git a/external/yq/pkg/yqlib/operator_delete_test.go b/external/yq/pkg/yqlib/operator_delete_test.go deleted file mode 100755 index afc938c..0000000 --- a/external/yq/pkg/yqlib/operator_delete_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var deleteOperatorScenarios = []expressionScenario{ - { - description: "Delete entry in map", - document: `{a: cat, b: dog}`, - expression: `del(.b)`, - expected: []string{ - "D0, P[], (doc)::{a: cat}\n", - }, - }, - { - description: "Delete nested entry in map", - document: `{a: {a1: fred, a2: frood}}`, - expression: `del(.a.a1)`, - expected: []string{ - "D0, P[], (doc)::{a: {a2: frood}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {a1: fred, a2: frood}}`, - expression: `.a | del(.a1)`, - expected: []string{ - "D0, P[a], (!!map)::{a2: frood}\n", - }, - }, - { - skipDoc: true, - description: "delete whole document", - document2: `a: slow`, - document: `a: fast`, - expression: `del(select(.a == "fast"))`, - expected: []string{ - "D0, P[], (doc)::a: slow\n", - }, - }, - { - skipDoc: true, - document: `a: [1,2,3]`, - expression: `.a | del(.[1])`, - expected: []string{ - "D0, P[a], (!!seq)::[1, 3]\n", - }, - }, - { - skipDoc: true, - document: `[0, {a: cat, b: dog}]`, - expression: `.[1] | del(.a)`, - expected: []string{ - "D0, P[1], (!!map)::{b: dog}\n", - }, - }, - { - skipDoc: true, - document: `[{a: cat, b: dog}]`, - expression: `.[0] | del(.a)`, - expected: []string{ - "D0, P[0], (!!map)::{b: dog}\n", - }, - }, - { - skipDoc: true, - document: `[{a: {b: thing, c: frog}}]`, - expression: `.[0].a | del(.b)`, - expected: []string{ - "D0, P[0 a], (!!map)::{c: frog}\n", - }, - }, - { - skipDoc: true, - document: `[{a: {b: thing, c: frog}}]`, - expression: `.[0] | del(.a.b)`, - expected: []string{ - "D0, P[0], (!!map)::{a: {c: frog}}\n", - }, - }, - { - skipDoc: true, - document: `{a: [0, {b: thing, c: frog}]}`, - expression: `.a[1] | del(.b)`, - expected: []string{ - "D0, P[a 1], (!!map)::{c: frog}\n", - }, - }, - { - skipDoc: true, - document: `{a: [0, {b: thing, c: frog}]}`, - expression: `.a | del(.[1].b)`, - expected: []string{ - "D0, P[a], (!!seq)::[0, {c: frog}]\n", - }, - }, - { - skipDoc: true, - document: `{a: {a1: fred, a2: frood}}`, - expression: `del(.. | select(.=="frood"))`, - expected: []string{ - "D0, P[], (!!map)::{a: {a1: fred}}\n", - }, - }, - { - description: "Delete entry in array", - document: `[1,2,3]`, - expression: `del(.[1])`, - expected: []string{ - "D0, P[], (doc)::[1, 3]\n", - }, - }, - { - skipDoc: true, - document: `a: [1,2,3]`, - expression: `del(.a[])`, - expected: []string{ - "D0, P[], (doc)::a: []\n", - }, - }, - { - skipDoc: true, - document: `a: [10,x,10, 10, x, 10]`, - expression: `del(.a[] | select(. == 10))`, - expected: []string{ - "D0, P[], (doc)::a: [x, x]\n", - }, - }, - { - skipDoc: true, - document: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`, - expression: `del(..)`, - expected: []string{ - "D0, P[], (!!map)::{}\n", - }, - }, - { - skipDoc: true, - document: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`, - expression: `del(.. | select(tag == "!!map") | (.b.thing1,.thing2))`, - expected: []string{ - "D0, P[], (!!map)::a: {thing1: yep, thing3: hi, b: {great: huh}}\n", - }, - }, - { - description: "Delete nested entry in array", - document: `[{a: cat, b: dog}]`, - expression: `del(.[0].a)`, - expected: []string{ - "D0, P[], (doc)::[{b: dog}]\n", - }, - }, - { - description: "Delete no matches", - document: `{a: cat, b: dog}`, - expression: `del(.c)`, - expected: []string{ - "D0, P[], (doc)::{a: cat, b: dog}\n", - }, - }, - { - description: "Delete matching entries", - document: `{a: cat, b: dog, c: bat}`, - expression: `del( .[] | select(. == "*at") )`, - expected: []string{ - "D0, P[], (doc)::{b: dog}\n", - }, - }, - { - description: "Recursively delete matching keys", - document: `{a: {name: frog, b: {name: blog, age: 12}}}`, - expression: `del(.. | select(has("name")).name)`, - expected: []string{ - "D0, P[], (!!map)::{a: {b: {age: 12}}}\n", - }, - }, -} - -func TestDeleteOperatorScenarios(t *testing.T) { - for _, tt := range deleteOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "delete", deleteOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_document_index.go b/external/yq/pkg/yqlib/operator_document_index.go deleted file mode 100755 index 566fe9d..0000000 --- a/external/yq/pkg/yqlib/operator_document_index.go +++ /dev/null @@ -1,20 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "gopkg.in/yaml.v3" -) - -func getDocumentIndexOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.Document), Tag: "!!int"} - scalar := candidate.CreateReplacement(node) - results.PushBack(scalar) - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_document_index_test.go b/external/yq/pkg/yqlib/operator_document_index_test.go deleted file mode 100755 index e0d8268..0000000 --- a/external/yq/pkg/yqlib/operator_document_index_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var documentIndexScenarios = []expressionScenario{ - { - description: "Retrieve a document index", - document: "a: cat\n---\na: frog\n", - expression: `.a | document_index`, - expected: []string{ - "D0, P[a], (!!int)::0\n", - "D1, P[a], (!!int)::1\n", - }, - }, - { - description: "Retrieve a document index, shorthand", - document: "a: cat\n---\na: frog\n", - expression: `.a | di`, - expected: []string{ - "D0, P[a], (!!int)::0\n", - "D1, P[a], (!!int)::1\n", - }, - }, - { - description: "Filter by document index", - document: "a: cat\n---\na: frog\n", - expression: `select(document_index == 1)`, - expected: []string{ - "D1, P[], (doc)::a: frog\n", - }, - }, - { - description: "Filter by document index shorthand", - document: "a: cat\n---\na: frog\n", - expression: `select(di == 1)`, - expected: []string{ - "D1, P[], (doc)::a: frog\n", - }, - }, - { - description: "Print Document Index with matches", - document: "a: cat\n---\na: frog\n", - expression: `.a | ({"match": ., "doc": document_index})`, - expected: []string{ - "D0, P[], (!!map)::match: cat\ndoc: 0\n", - "D0, P[], (!!map)::match: frog\ndoc: 1\n", - }, - }, -} - -func TestDocumentIndexScenarios(t *testing.T) { - for _, tt := range documentIndexScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "document-index", documentIndexScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_encoder_decoder.go b/external/yq/pkg/yqlib/operator_encoder_decoder.go deleted file mode 100755 index 8f41389..0000000 --- a/external/yq/pkg/yqlib/operator_encoder_decoder.go +++ /dev/null @@ -1,141 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "container/list" - "regexp" - "strings" - - "gopkg.in/yaml.v3" -) - -func configureEncoder(format PrinterOutputFormat, indent int) Encoder { - switch format { - case JSONOutputFormat: - return NewJSONEncoder(indent, false, false) - case PropsOutputFormat: - return NewPropertiesEncoder(true) - case CSVOutputFormat: - return NewCsvEncoder(',') - case TSVOutputFormat: - return NewCsvEncoder('\t') - case YamlOutputFormat: - return NewYamlEncoder(indent, false, ConfiguredYamlPreferences) - case XMLOutputFormat: - return NewXMLEncoder(indent, ConfiguredXMLPreferences) - case Base64OutputFormat: - return NewBase64Encoder() - } - panic("invalid encoder") -} - -func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { - var output bytes.Buffer - log.Debug("printing with indent: %v", prefs.indent) - - encoder := configureEncoder(prefs.format, prefs.indent) - - printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output))) - err := printer.PrintResults(candidate.AsList()) - return output.String(), err -} - -type encoderPreferences struct { - format PrinterOutputFormat - indent int -} - -/* encodes object as yaml string */ - -func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - preferences := expressionNode.Operation.Preferences.(encoderPreferences) - var results = list.New() - - hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$") - endWithNewLine := regexp.MustCompile(".*\n$") - chomper := regexp.MustCompile("\n+$") - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - stringValue, err := encodeToString(candidate, preferences) - - if err != nil { - return Context{}, err - } - - // remove trailing newlines if needed. - // check if we originally decoded this path, and the original thing had a single line. - originalList := context.GetVariable("decoded: " + candidate.GetKey()) - if originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) { - - original := originalList.Front().Value.(*CandidateNode) - originalNode := unwrapDoc(original.Node) - // original block did not have a new line at the end, get rid of this one too - if !endWithNewLine.MatchString(originalNode.Value) { - stringValue = chomper.ReplaceAllString(stringValue, "") - } - } - - // dont print a new line when printing json on a single line. - if (preferences.format == JSONOutputFormat && preferences.indent == 0) || - preferences.format == CSVOutputFormat || - preferences.format == TSVOutputFormat { - stringValue = chomper.ReplaceAllString(stringValue, "") - } - - stringContentNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: stringValue} - results.PushBack(candidate.CreateReplacement(stringContentNode)) - } - return context.ChildContext(results), nil -} - -type decoderPreferences struct { - format InputFormat -} - -/* takes a string and decodes it back into an object */ -func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - preferences := expressionNode.Operation.Preferences.(decoderPreferences) - - var decoder Decoder - switch preferences.format { - case YamlInputFormat: - decoder = NewYamlDecoder(ConfiguredYamlPreferences) - case XMLInputFormat: - decoder = NewXMLDecoder(ConfiguredXMLPreferences) - case Base64InputFormat: - decoder = NewBase64Decoder() - case PropertiesInputFormat: - decoder = NewPropertiesDecoder() - case CSVObjectInputFormat: - decoder = NewCSVObjectDecoder(',') - case TSVObjectInputFormat: - decoder = NewCSVObjectDecoder('\t') - } - - var results = list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList()) - - log.Debugf("got: [%v]", candidate.Node.Value) - - err := decoder.Init(strings.NewReader(unwrapDoc(candidate.Node).Value)) - if err != nil { - return Context{}, err - } - - decodedNode, errorReading := decoder.Decode() - if errorReading != nil { - return Context{}, errorReading - } - //first node is a doc - node := unwrapDoc(decodedNode.Node) - - results.PushBack(candidate.CreateReplacement(node)) - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_encoder_decoder_test.go b/external/yq/pkg/yqlib/operator_encoder_decoder_test.go deleted file mode 100755 index e3961fb..0000000 --- a/external/yq/pkg/yqlib/operator_encoder_decoder_test.go +++ /dev/null @@ -1,285 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var prefix = "D0, P[], (doc)::a:\n cool:\n bob: dylan\n" - -var encoderDecoderOperatorScenarios = []expressionScenario{ - { - description: "Encode value as json string", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_json)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"} -`, - }, - }, - { - description: "Encode value as json string, on one line", - subdescription: "Pass in a 0 indent to print json on a single line.", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_json(0))`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} -`, - }, - }, - { - description: "Encode value as json string, on one line shorthand", - subdescription: "Pass in a 0 indent to print json on a single line.", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | @json)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} -`, - }, - }, - { - description: "Decode a json encoded string", - subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.", - document: `a: '{"cool":"thing"}'`, - expression: `.a | from_json | ... style=""`, - expected: []string{ - "D0, P[a], (!!map)::cool: thing\n", - }, - }, - { - skipDoc: true, - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_props)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"} -`, - }, - }, - { - description: "Encode value as props string", - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | @props)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"} -`, - }, - }, - { - description: "Decode props encoded string", - document: `a: "cats=great\ndogs=cool as well"`, - expression: `.a |= @propsd`, - expected: []string{ - "D0, P[], (doc)::a:\n cats: great\n dogs: cool as well\n", - }, - }, - { - description: "Decode csv encoded string", - document: `a: "cats,dogs\ngreat,cool as well"`, - expression: `.a |= @csvd`, - expected: []string{ - "D0, P[], (doc)::a:\n - cats: great\n dogs: cool as well\n", - }, - }, - { - description: "Decode tsv encoded string", - document: `a: "cats dogs\ngreat cool as well"`, - expression: `.a |= @tsvd`, - expected: []string{ - "D0, P[], (doc)::a:\n - cats: great\n dogs: cool as well\n", - }, - }, - { - skipDoc: true, - document: "a:\n cool:\n bob: dylan", - expression: `.b = (.a | @yaml)`, - expected: []string{ - prefix + "b: |\n cool:\n bob: dylan\n", - }, - }, - { - description: "Encode value as yaml string", - subdescription: "Indent defaults to 2", - document: "a:\n cool:\n bob: dylan", - expression: `.b = (.a | to_yaml)`, - expected: []string{ - prefix + "b: |\n cool:\n bob: dylan\n", - }, - }, - { - description: "Encode value as yaml string, with custom indentation", - subdescription: "You can specify the indentation level as the first parameter.", - document: "a:\n cool:\n bob: dylan", - expression: `.b = (.a | to_yaml(8))`, - expected: []string{ - prefix + "b: |\n cool:\n bob: dylan\n", - }, - }, - { - skipDoc: true, - document: `{a: {cool: "thing"}}`, - expression: `.b = (.a | to_yaml)`, - expected: []string{ - `D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"} -`, - }, - }, - { - description: "Decode a yaml encoded string", - document: `a: "foo: bar"`, - expression: `.b = (.a | from_yaml)`, - expected: []string{ - "D0, P[], (doc)::a: \"foo: bar\"\nb:\n foo: bar\n", - }, - }, - { - description: "Update a multiline encoded yaml string", - dontFormatInputForDoc: true, - document: "a: |\n foo: bar\n baz: dog\n", - expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, - expected: []string{ - "D0, P[], (doc)::a: |\n foo: cat\n baz: dog\n", - }, - }, - { - skipDoc: true, - dontFormatInputForDoc: true, - document: "a: |-\n foo: bar\n baz: dog\n", - expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, - expected: []string{ - "D0, P[], (doc)::a: |-\n foo: cat\n baz: dog\n", - }, - }, - { - description: "Update a single line encoded yaml string", - dontFormatInputForDoc: true, - document: "a: 'foo: bar'", - expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, - expected: []string{ - "D0, P[], (doc)::a: 'foo: cat'\n", - }, - }, - { - description: "Encode array of scalars as csv string", - subdescription: "Scalars are strings, numbers and booleans.", - document: `[cat, "thing1,thing2", true, 3.40]`, - expression: `@csv`, - expected: []string{ - "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n", - }, - }, - { - description: "Encode array of arrays as csv string", - document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, - expression: `@csv`, - expected: []string{ - "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n", - }, - }, - { - description: "Encode array of array scalars as tsv string", - subdescription: "Scalars are strings, numbers and booleans.", - document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, - expression: `@tsv`, - expected: []string{ - "D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n", - }, - }, - { - skipDoc: true, - dontFormatInputForDoc: true, - document: "a: \"foo: bar\"", - expression: `.a |= (from_yaml | .foo = {"a": "frog"} | to_yaml)`, - expected: []string{ - "D0, P[], (doc)::a: \"foo:\\n a: frog\"\n", - }, - }, - { - description: "Encode value as xml string", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `.a | to_xml`, - expected: []string{ - "D0, P[a], (!!str)::\n bar\n\n\n", - }, - }, - { - description: "Encode value as xml string on a single line", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `.a | @xml`, - expected: []string{ - "D0, P[a], (!!str)::bar\n\n", - }, - }, - { - description: "Encode value as xml string with custom indentation", - document: `{a: {cool: {foo: "bar", +@id: hi}}}`, - expression: `{"cat": .a | to_xml(1)}`, - expected: []string{ - "D0, P[], (!!map)::cat: |\n \n bar\n \n", - }, - }, - { - description: "Decode a xml encoded string", - document: `a: "bar"`, - expression: `.b = (.a | from_xml)`, - expected: []string{ - "D0, P[], (doc)::a: \"bar\"\nb:\n foo: bar\n", - }, - }, - { - description: "Encode a string to base64", - document: "coolData: a special string", - expression: ".coolData | @base64", - expected: []string{ - "D0, P[coolData], (!!str)::YSBzcGVjaWFsIHN0cmluZw==\n", - }, - }, - { - description: "Encode a yaml document to base64", - subdescription: "Pipe through @yaml first to convert to a string, then use @base64 to encode it.", - document: "a: apple", - expression: "@yaml | @base64", - expected: []string{ - "D0, P[], (!!str)::YTogYXBwbGUK\n", - }, - }, - { - description: "Decode a base64 encoded string", - subdescription: "Decoded data is assumed to be a string.", - document: "coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==", - expression: ".coolData | @base64d", - expected: []string{ - "D0, P[coolData], (!!str)::Works with UTF-16 😊\n", - }, - }, - { - description: "Decode a base64 encoded yaml document", - subdescription: "Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.", - document: "coolData: YTogYXBwbGUK", - expression: ".coolData |= (@base64d | from_yaml)", - expected: []string{ - "D0, P[], (doc)::coolData:\n a: apple\n", - }, - }, - { - description: "empty base64 decode", - skipDoc: true, - expression: `"" | @base64d`, - expected: []string{ - "D0, P[], (!!str)::\n", - }, - }, - { - description: "empty xml decode", - skipDoc: true, - expression: `"" | @xmld`, - expected: []string{ - "D0, P[], (!!null)::\n", - }, - }, -} - -func TestEncoderDecoderOperatorScenarios(t *testing.T) { - for _, tt := range encoderDecoderOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "encode-decode", encoderDecoderOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_entries.go b/external/yq/pkg/yqlib/operator_entries.go deleted file mode 100755 index 2af9fe9..0000000 --- a/external/yq/pkg/yqlib/operator_entries.go +++ /dev/null @@ -1,180 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node { - var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"} - var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"} - - return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", - Content: []*yaml.Node{keyKey, key, valueKey, value}, - } -} - -func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { - var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence) - - var contents = unwrapDoc(candidateNode.Node).Content - for index := 0; index < len(contents); index = index + 2 { - key := contents[index] - value := contents[index+1] - - sequence.Content = append(sequence.Content, entrySeqFor(key, value)) - } - return entriesNode -} - -func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode { - var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence) - - var contents = unwrapDoc(candidateNode.Node).Content - for index := 0; index < len(contents); index = index + 1 { - key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)} - value := contents[index] - - sequence.Content = append(sequence.Content, entrySeqFor(key, value)) - } - return entriesNode -} - -func toEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - - switch candidateNode.Kind { - case yaml.MappingNode: - results.PushBack(toEntriesFromMap(candidate)) - - case yaml.SequenceNode: - results.PushBack(toEntriesfromSeq(candidate)) - default: - if candidateNode.Tag != "!!null" { - return Context{}, fmt.Errorf("%v has no keys", candidate.Node.Tag) - } - } - } - - return context.ChildContext(results), nil -} - -func parseEntry(entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error) { - prefs := traversePreferences{DontAutoCreate: true} - candidateNode := &CandidateNode{Node: entry} - - keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false) - - if err != nil { - return nil, nil, err - } else if keyResults.Len() != 1 { - return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position) - } - - valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false) - - if err != nil { - return nil, nil, err - } else if valueResults.Len() != 1 { - return nil, nil, fmt.Errorf("expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position) - } - - return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil - -} - -func fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) { - var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - var mapCandidateNode = candidateNode.CreateReplacementWithDocWrappers(node) - - var contents = unwrapDoc(candidateNode.Node).Content - - for index := 0; index < len(contents); index = index + 1 { - key, value, err := parseEntry(contents[index], index) - if err != nil { - return nil, err - } - - node.Content = append(node.Content, key, value) - } - return mapCandidateNode, nil -} - -func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - - switch candidateNode.Kind { - case yaml.SequenceNode: - mapResult, err := fromEntries(candidate) - if err != nil { - return Context{}, err - } - results.PushBack(mapResult) - default: - return Context{}, fmt.Errorf("from entries only runs against arrays") - } - } - - return context.ChildContext(results), nil -} - -func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - //to_entries on the context - toEntries, err := toEntriesOperator(d, context, expressionNode) - if err != nil { - return Context{}, err - } - - var results = list.New() - - for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - //run expression against entries - // splat toEntries and pipe it into Rhs - splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) - if err != nil { - return Context{}, err - } - - result, err := d.GetMatchingNodes(splatted, expressionNode.RHS) - log.Debug("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType) - log.Debug("result %v", result) - if err != nil { - return Context{}, err - } - - selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} - collected, err := collectTogether(d, result, selfExpression) - if err != nil { - return Context{}, err - } - collected.LeadingContent = candidate.LeadingContent - collected.TrailingContent = candidate.TrailingContent - - log.Debugf("**** collected %v", collected.LeadingContent) - - fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode) - if err != nil { - return Context{}, err - } - results.PushBackList(fromEntries.MatchingNodes) - - } - - //from_entries on the result - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_entries_test.go b/external/yq/pkg/yqlib/operator_entries_test.go deleted file mode 100755 index 9313323..0000000 --- a/external/yq/pkg/yqlib/operator_entries_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var entriesOperatorScenarios = []expressionScenario{ - { - description: "to_entries Map", - document: `{a: 1, b: 2}`, - expression: `to_entries`, - expected: []string{ - "D0, P[], (!!seq)::- key: a\n value: 1\n- key: b\n value: 2\n", - }, - }, - { - description: "to_entries Array", - document: `[a, b]`, - expression: `to_entries`, - expected: []string{ - "D0, P[], (!!seq)::- key: 0\n value: a\n- key: 1\n value: b\n", - }, - }, - { - description: "to_entries null", - document: `null`, - expression: `to_entries`, - expected: []string{}, - }, - { - description: "from_entries map", - document: `{a: 1, b: 2}`, - expression: `to_entries | from_entries`, - expected: []string{ - "D0, P[], (!!map)::a: 1\nb: 2\n", - }, - }, - { - description: "from_entries with numeric key indexes", - subdescription: "from_entries always creates a map, even for numeric keys", - document: `[a,b]`, - expression: `to_entries | from_entries`, - expected: []string{ - "D0, P[], (!!map)::0: a\n1: b\n", - }, - }, - { - description: "Use with_entries to update keys", - document: `{a: 1, b: 2}`, - expression: `with_entries(.key |= "KEY_" + .)`, - expected: []string{ - "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", - }, - }, - { - description: "Custom sort map keys", - subdescription: "Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.", - document: `{a: 1, c: 3, b: 2}`, - expression: `to_entries | sort_by(.key) | reverse | from_entries`, - expected: []string{ - "D0, P[], (!!map)::c: 3\nb: 2\na: 1\n", - }, - }, - { - skipDoc: true, - document: `{a: 1, b: 2}`, - document2: `{c: 1, d: 2}`, - expression: `with_entries(.key |= "KEY_" + .)`, - expected: []string{ - "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", - "D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n", - }, - }, - { - skipDoc: true, - document: `[{a: 1, b: 2}, {c: 1, d: 2}]`, - expression: `.[] | with_entries(.key |= "KEY_" + .)`, - expected: []string{ - "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", - "D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n", - }, - }, - { - description: "Use with_entries to filter the map", - document: `{a: { b: bird }, c: { d: dog }}`, - expression: `with_entries(select(.value | has("b")))`, - expected: []string{ - "D0, P[], (!!map)::a: {b: bird}\n", - }, - }, - { - description: "Use with_entries to filter the map; head comment", - skipDoc: true, - document: "# abc\n{a: { b: bird }, c: { d: dog }}\n# xyz", - expression: `with_entries(select(.value | has("b")))`, - expected: []string{ - "D0, P[], (!!map)::# abc\na: {b: bird}\n# xyz\n", - }, - }, -} - -func TestEntriesOperatorScenarios(t *testing.T) { - for _, tt := range entriesOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "entries", entriesOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_env.go b/external/yq/pkg/yqlib/operator_env.go deleted file mode 100755 index ed3f51e..0000000 --- a/external/yq/pkg/yqlib/operator_env.go +++ /dev/null @@ -1,90 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "os" - "strings" - - parse "github.com/a8m/envsubst/parse" - yaml "gopkg.in/yaml.v3" -) - -type envOpPreferences struct { - StringValue bool - NoUnset bool - NoEmpty bool - FailFast bool -} - -func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - envName := expressionNode.Operation.CandidateNode.Node.Value - log.Debug("EnvOperator, env name:", envName) - - rawValue := os.Getenv(envName) - - preferences := expressionNode.Operation.Preferences.(envOpPreferences) - - var node *yaml.Node - if preferences.StringValue { - node = &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: rawValue, - } - } else if rawValue == "" { - return Context{}, fmt.Errorf("Value for env variable '%v' not provided in env()", envName) - } else { - var dataBucket yaml.Node - decoder := yaml.NewDecoder(strings.NewReader(rawValue)) - errorReading := decoder.Decode(&dataBucket) - if errorReading != nil { - return Context{}, errorReading - } - //first node is a doc - node = unwrapDoc(&dataBucket) - } - log.Debug("ENV tag", node.Tag) - log.Debug("ENV value", node.Value) - log.Debug("ENV Kind", node.Kind) - - target := &CandidateNode{Node: node} - - return context.SingleChildContext(target), nil -} - -func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - preferences := envOpPreferences{} - if expressionNode.Operation.Preferences != nil { - preferences = expressionNode.Operation.Preferences.(envOpPreferences) - } - - parser := parse.New("string", os.Environ(), - &parse.Restrictions{NoUnset: preferences.NoUnset, NoEmpty: preferences.NoEmpty}) - - if preferences.FailFast { - parser.Mode = parse.Quick - } else { - parser.Mode = parse.AllErrors - } - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - if node.Tag != "!!str" { - log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value) - return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) - } - - value, err := parser.Parse(node.Value) - if err != nil { - return Context{}, err - } - targetNode := &yaml.Node{Kind: yaml.ScalarNode, Value: value, Tag: "!!str"} - result := candidate.CreateReplacement(targetNode) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_env_test.go b/external/yq/pkg/yqlib/operator_env_test.go deleted file mode 100755 index 7f403dd..0000000 --- a/external/yq/pkg/yqlib/operator_env_test.go +++ /dev/null @@ -1,170 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var envOperatorScenarios = []expressionScenario{ - { - description: "Read string environment variable", - environmentVariables: map[string]string{"myenv": "cat meow"}, - expression: `.a = env(myenv)`, - expected: []string{ - "D0, P[], ()::a: cat meow\n", - }, - }, - { - description: "Read boolean environment variable", - environmentVariables: map[string]string{"myenv": "true"}, - expression: `.a = env(myenv)`, - expected: []string{ - "D0, P[], ()::a: true\n", - }, - }, - { - description: "Read numeric environment variable", - environmentVariables: map[string]string{"myenv": "12"}, - expression: `.a = env(myenv)`, - expected: []string{ - "D0, P[], ()::a: 12\n", - }, - }, - { - description: "Read yaml environment variable", - environmentVariables: map[string]string{"myenv": "{b: fish}"}, - expression: `.a = env(myenv)`, - expected: []string{ - "D0, P[], ()::a: {b: fish}\n", - }, - }, - { - description: "Read boolean environment variable as a string", - environmentVariables: map[string]string{"myenv": "true"}, - expression: `.a = strenv(myenv)`, - expected: []string{ - "D0, P[], ()::a: \"true\"\n", - }, - }, - { - description: "Read numeric environment variable as a string", - environmentVariables: map[string]string{"myenv": "12"}, - expression: `.a = strenv(myenv)`, - expected: []string{ - "D0, P[], ()::a: \"12\"\n", - }, - }, - { - description: "Dynamically update a path from an environment variable", - subdescription: "The env variable can be any valid yq expression.", - document: `{a: {b: [{name: dog}, {name: cat}]}}`, - environmentVariables: map[string]string{"pathEnv": ".a.b[0].name", "valueEnv": "moo"}, - expression: `eval(strenv(pathEnv)) = strenv(valueEnv)`, - expected: []string{ - "D0, P[], (doc)::{a: {b: [{name: moo}, {name: cat}]}}\n", - }, - }, - { - description: "Dynamic key lookup with environment variable", - environmentVariables: map[string]string{"myenv": "cat"}, - document: `{cat: meow, dog: woof}`, - expression: `.[env(myenv)]`, - expected: []string{ - "D0, P[cat], (!!str)::meow\n", - }, - }, - { - description: "Replace strings with envsubst", - environmentVariables: map[string]string{"myenv": "cat"}, - expression: `"the ${myenv} meows" | envsubst`, - expected: []string{ - "D0, P[], (!!str)::the cat meows\n", - }, - }, - { - description: "Replace strings with envsubst, missing variables", - expression: `"the ${myenvnonexisting} meows" | envsubst`, - expected: []string{ - "D0, P[], (!!str)::the meows\n", - }, - }, - { - description: "Replace strings with envsubst(nu), missing variables", - subdescription: "(nu) not unset, will fail if there are unset (missing) variables", - expression: `"the ${myenvnonexisting} meows" | envsubst(nu)`, - expectedError: "variable ${myenvnonexisting} not set", - }, - { - description: "Replace strings with envsubst(ne), missing variables", - subdescription: "(ne) not empty, only validates set variables", - expression: `"the ${myenvnonexisting} meows" | envsubst(ne)`, - expected: []string{ - "D0, P[], (!!str)::the meows\n", - }, - }, - { - description: "Replace strings with envsubst(ne), empty variable", - subdescription: "(ne) not empty, will fail if a references variable is empty", - environmentVariables: map[string]string{"myenv": ""}, - expression: `"the ${myenv} meows" | envsubst(ne)`, - expectedError: "variable ${myenv} set but empty", - }, - { - description: "Replace strings with envsubst, missing variables with defaults", - expression: `"the ${myenvnonexisting-dog} meows" | envsubst`, - expected: []string{ - "D0, P[], (!!str)::the dog meows\n", - }, - }, - { - description: "Replace strings with envsubst(nu), missing variables with defaults", - subdescription: "Having a default specified skips over the missing variable.", - expression: `"the ${myenvnonexisting-dog} meows" | envsubst(nu)`, - expected: []string{ - "D0, P[], (!!str)::the dog meows\n", - }, - }, - { - description: "Replace strings with envsubst(ne), missing variables with defaults", - subdescription: "Fails, because the variable is explicitly set to blank.", - environmentVariables: map[string]string{"myEmptyEnv": ""}, - expression: `"the ${myEmptyEnv-dog} meows" | envsubst(ne)`, - expectedError: "variable ${myEmptyEnv} set but empty", - }, - { - description: "Replace string environment variable in document", - environmentVariables: map[string]string{"myenv": "cat meow"}, - document: "{v: \"${myenv}\"}", - expression: `.v |= envsubst`, - expected: []string{ - "D0, P[], (doc)::{v: \"cat meow\"}\n", - }, - }, - { - description: "(Default) Return all envsubst errors", - subdescription: "By default, all errors are returned at once.", - expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu)`, - expectedError: "variable ${notThere} not set\nvariable ${alsoNotThere} not set", - }, - { - description: "Fail fast, return the first envsubst error (and abort)", - expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`, - expectedError: "variable ${notThere} not set", - }, - { - description: "with header/footer", - skipDoc: true, - environmentVariables: map[string]string{"myenv": "cat meow"}, - document: "# abc\n{v: \"${myenv}\"}\n# xyz\n", - expression: `(.. | select(tag == "!!str")) |= envsubst`, - expected: []string{ - "D0, P[], (!!map)::# abc\n{v: \"cat meow\"}\n# xyz\n", - }, - }, -} - -func TestEnvOperatorScenarios(t *testing.T) { - for _, tt := range envOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "env-variable-operators", envOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_equals.go b/external/yq/pkg/yqlib/operator_equals.go deleted file mode 100755 index 313c7a4..0000000 --- a/external/yq/pkg/yqlib/operator_equals.go +++ /dev/null @@ -1,54 +0,0 @@ -package yqlib - -import "gopkg.in/yaml.v3" - -func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- equalsOperation") - return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(false), true) -} - -func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - value := false - log.Debugf("-- isEquals cross function") - if lhs == nil && rhs == nil { - owner := &CandidateNode{} - return createBooleanCandidate(owner, !flip), nil - } else if lhs == nil { - log.Debugf("lhs nil, but rhs is not") - rhsNode := unwrapDoc(rhs.Node) - value := rhsNode.Tag == "!!null" - if flip { - value = !value - } - return createBooleanCandidate(rhs, value), nil - } else if rhs == nil { - log.Debugf("lhs not nil, but rhs is") - lhsNode := unwrapDoc(lhs.Node) - value := lhsNode.Tag == "!!null" - if flip { - value = !value - } - return createBooleanCandidate(lhs, value), nil - } - - lhsNode := unwrapDoc(lhs.Node) - rhsNode := unwrapDoc(rhs.Node) - - if lhsNode.Tag == "!!null" { - value = (rhsNode.Tag == "!!null") - } else if lhsNode.Kind == yaml.ScalarNode && rhsNode.Kind == yaml.ScalarNode { - value = matchKey(lhsNode.Value, rhsNode.Value) - } - log.Debugf("%v == %v ? %v", NodeToString(lhs), NodeToString(rhs), value) - if flip { - value = !value - } - return createBooleanCandidate(lhs, value), nil - } -} - -func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- notEqualsOperator") - return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true) -} diff --git a/external/yq/pkg/yqlib/operator_equals_test.go b/external/yq/pkg/yqlib/operator_equals_test.go deleted file mode 100755 index 70b9f88..0000000 --- a/external/yq/pkg/yqlib/operator_equals_test.go +++ /dev/null @@ -1,190 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var equalsOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - expression: ".a == .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `a: cat`, - expression: ".a == .b", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `a: cat`, - expression: ".b == .a", - expected: []string{ - "D0, P[a], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "cat", - document2: "dog", - expression: "select(fi==0) == select(fi==1)", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: "(.a == .b) as $x", - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: ".a == .b", - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: "(.a != .b) as $x", - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - skipDoc: true, - document: "{}", - expression: ".a != .b", - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: "{a: {b: 10}}", - expression: "select(.c != null)", - expected: []string{}, - }, - { - skipDoc: true, - document: "{a: {b: 10}}", - expression: "select(.d == .c)", - expected: []string{ - "D0, P[], (doc)::{a: {b: 10}}\n", - }, - }, - { - skipDoc: true, - document: "{a: {b: 10}}", - expression: "select(null == .c)", - expected: []string{ - "D0, P[], (doc)::{a: {b: 10}}\n", - }, - }, - { - skipDoc: true, - document: "{a: { b: {things: \"\"}, f: [1], g: [] }}", - expression: ".. | select(. == \"\")", - expected: []string{ - "D0, P[a b things], (!!str)::\n", - }, - }, - { - description: "Match string", - document: `[cat,goat,dog]`, - expression: `.[] | (. == "*at")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::true\n", - "D0, P[2], (!!bool)::false\n", - }, - }, - { - description: "Don't match string", - document: `[cat,goat,dog]`, - expression: `.[] | (. != "*at")`, - expected: []string{ - "D0, P[0], (!!bool)::false\n", - "D0, P[1], (!!bool)::false\n", - "D0, P[2], (!!bool)::true\n", - }, - }, - { - description: "Match number", - document: `[3, 4, 5]`, - expression: `.[] | (. == 4)`, - expected: []string{ - "D0, P[0], (!!bool)::false\n", - "D0, P[1], (!!bool)::true\n", - "D0, P[2], (!!bool)::false\n", - }, - }, - { - description: "Don't match number", - document: `[3, 4, 5]`, - expression: `.[] | (. != 4)`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::false\n", - "D0, P[2], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`, - expression: `.a | (.[].b == "apple")`, - expected: []string{ - "D0, P[a cat b], (!!bool)::true\n", - "D0, P[a pat b], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: ``, - expression: `null == null`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "Match nulls", - document: ``, - expression: `null == ~`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - description: "Non existent key doesn't equal a value", - document: "a: frog", - expression: `select(.b != "thing")`, - expected: []string{ - "D0, P[], (doc)::a: frog\n", - }, - }, - { - description: "Two non existent keys are equal", - document: "a: frog", - expression: `select(.b == .c)`, - expected: []string{ - "D0, P[], (doc)::a: frog\n", - }, - }, -} - -func TestEqualOperatorScenarios(t *testing.T) { - for _, tt := range equalsOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "equals", equalsOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_error.go b/external/yq/pkg/yqlib/operator_error.go deleted file mode 100755 index 8b0e3a9..0000000 --- a/external/yq/pkg/yqlib/operator_error.go +++ /dev/null @@ -1,20 +0,0 @@ -package yqlib - -import ( - "fmt" -) - -func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- errorOperation") - - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - errorMessage := "aborted" - if rhs.MatchingNodes.Len() > 0 { - errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - return Context{}, fmt.Errorf(errorMessage) -} diff --git a/external/yq/pkg/yqlib/operator_error_test.go b/external/yq/pkg/yqlib/operator_error_test.go deleted file mode 100755 index 845686d..0000000 --- a/external/yq/pkg/yqlib/operator_error_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package yqlib - -import "testing" - -const validationExpression = ` - with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | - .numPets = env(numberOfCats) -` - -var errorOperatorScenarios = []expressionScenario{ - { - description: "Validate a particular value", - document: `a: hello`, - expression: `select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")`, - expectedError: ".a [hello] is not howdy!", - }, - { - description: "Validate the environment variable is a number - invalid", - environmentVariables: map[string]string{"numberOfCats": "please"}, - expression: `env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")`, - expectedError: "numberOfCats is not a number :(", - }, - { - description: "Validate the environment variable is a number - valid", - subdescription: "`with` can be a convenient way of encapsulating validation.", - environmentVariables: map[string]string{"numberOfCats": "3"}, - document: "name: Bob\nfavouriteAnimal: cat\n", - expression: validationExpression, - expected: []string{ - "D0, P[], (doc)::name: Bob\nfavouriteAnimal: cat\nnumPets: 3\n", - }, - }, -} - -func TestErrorOperatorScenarios(t *testing.T) { - for _, tt := range errorOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "error", errorOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_eval.go b/external/yq/pkg/yqlib/operator_eval.go deleted file mode 100755 index 78037e9..0000000 --- a/external/yq/pkg/yqlib/operator_eval.go +++ /dev/null @@ -1,42 +0,0 @@ -package yqlib - -import ( - "container/list" -) - -func evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("Eval") - pathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - expressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len()) - expIndex := 0 - //parse every expression - for pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() { - expressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode) - - expressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Node.Value) - if err != nil { - return Context{}, err - } - - expIndex++ - } - - results := list.New() - - for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { - for expIndex = 0; expIndex < len(expressions); expIndex++ { - result, err := d.GetMatchingNodes(context, expressions[expIndex]) - if err != nil { - return Context{}, err - } - results.PushBackList(result.MatchingNodes) - } - } - - return context.ChildContext(results), nil - -} diff --git a/external/yq/pkg/yqlib/operator_eval_test.go b/external/yq/pkg/yqlib/operator_eval_test.go deleted file mode 100755 index 3bff5a1..0000000 --- a/external/yq/pkg/yqlib/operator_eval_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var evalOperatorScenarios = []expressionScenario{ - - { - description: "Dynamically evaluate a path", - document: `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`, - expression: `eval(.pathExp)`, - expected: []string{ - "D0, P[a b 1], (!!map)::{name: cat}\n", - }, - }, - { - description: "Dynamically update a path from an environment variable", - subdescription: "The env variable can be any valid yq expression.", - document: `{a: {b: [{name: dog}, {name: cat}]}}`, - environmentVariables: map[string]string{"pathEnv": ".a.b[0].name", "valueEnv": "moo"}, - expression: `eval(strenv(pathEnv)) = strenv(valueEnv)`, - expected: []string{ - "D0, P[], (doc)::{a: {b: [{name: moo}, {name: cat}]}}\n", - }, - }, -} - -func TestEvalOperatorsScenarios(t *testing.T) { - for _, tt := range evalOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "eval", evalOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_expression.go b/external/yq/pkg/yqlib/operator_expression.go deleted file mode 100755 index e451f0e..0000000 --- a/external/yq/pkg/yqlib/operator_expression.go +++ /dev/null @@ -1,16 +0,0 @@ -package yqlib - -type expressionOpPreferences struct { - expression string -} - -func expressionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - prefs := expressionNode.Operation.Preferences.(expressionOpPreferences) - expNode, err := ExpressionParser.ParseExpression(prefs.expression) - if err != nil { - return Context{}, err - } - - return d.GetMatchingNodes(context, expNode) -} diff --git a/external/yq/pkg/yqlib/operator_file.go b/external/yq/pkg/yqlib/operator_file.go deleted file mode 100755 index c0ffaad..0000000 --- a/external/yq/pkg/yqlib/operator_file.go +++ /dev/null @@ -1,38 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func getFilenameOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetFilename") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Filename, Tag: "!!str"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} - -func getFileIndexOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetFileIndex") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.FileIndex), Tag: "!!int"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_file_test.go b/external/yq/pkg/yqlib/operator_file_test.go deleted file mode 100755 index 3ad7224..0000000 --- a/external/yq/pkg/yqlib/operator_file_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var fileOperatorScenarios = []expressionScenario{ - { - description: "Get filename", - document: `{a: cat}`, - expression: `filename`, - expected: []string{ - "D0, P[], (!!str)::sample.yml\n", - }, - }, - { - description: "Get file index", - document: `{a: cat}`, - expression: `file_index`, - expected: []string{ - "D0, P[], (!!int)::0\n", - }, - }, - { - description: "Get file indices of multiple documents", - document: `{a: cat}`, - document2: `{a: cat}`, - expression: `file_index`, - expected: []string{ - "D0, P[], (!!int)::0\n", - "D0, P[], (!!int)::1\n", - }, - }, - { - description: "Get file index alias", - document: `{a: cat}`, - expression: `fi`, - expected: []string{ - "D0, P[], (!!int)::0\n", - }, - }, - { - skipDoc: true, - document: "a: cat\nb: dog", - expression: `.. lineComment |= filename`, - expected: []string{ - "D0, P[], (!!map)::a: cat # sample.yml\nb: dog # sample.yml\n", - }, - }, -} - -func TestFileOperatorsScenarios(t *testing.T) { - for _, tt := range fileOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "file-operators", fileOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_flatten.go b/external/yq/pkg/yqlib/operator_flatten.go deleted file mode 100755 index e7ccf99..0000000 --- a/external/yq/pkg/yqlib/operator_flatten.go +++ /dev/null @@ -1,54 +0,0 @@ -package yqlib - -import ( - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -type flattenPreferences struct { - depth int -} - -func flatten(node *yaml.Node, depth int) { - if depth == 0 { - return - } - if node.Kind != yaml.SequenceNode { - return - } - content := node.Content - newSeq := make([]*yaml.Node, 0) - - for i := 0; i < len(content); i++ { - if content[i].Kind == yaml.SequenceNode { - flatten(content[i], depth-1) - for j := 0; j < len(content[i].Content); j++ { - newSeq = append(newSeq, content[i].Content[j]) - } - } else { - newSeq = append(newSeq, content[i]) - } - } - node.Content = newSeq -} - -func flattenOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- flatten Operator") - depth := expressionNode.Operation.Preferences.(flattenPreferences).depth - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - if candidateNode.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("Only arrays are supported for flatten") - } - - flatten(candidateNode, depth) - - } - - return context, nil - -} diff --git a/external/yq/pkg/yqlib/operator_flatten_test.go b/external/yq/pkg/yqlib/operator_flatten_test.go deleted file mode 100755 index add71e1..0000000 --- a/external/yq/pkg/yqlib/operator_flatten_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var flattenOperatorScenarios = []expressionScenario{ - { - description: "Flatten", - subdescription: "Recursively flattens all arrays", - document: `[1, [2], [[3]]]`, - expression: `flatten`, - expected: []string{ - "D0, P[], (doc)::[1, 2, 3]\n", - }, - }, - { - description: "Flatten with depth of one", - document: `[1, [2], [[3]]]`, - expression: `flatten(1)`, - expected: []string{ - "D0, P[], (doc)::[1, 2, [3]]\n", - }, - }, - { - description: "Flatten empty array", - document: `[[]]`, - expression: `flatten`, - expected: []string{ - "D0, P[], (doc)::[]\n", - }, - }, - { - description: "Flatten array of objects", - document: `[{foo: bar}, [{foo: baz}]]`, - expression: `flatten`, - expected: []string{ - "D0, P[], (doc)::[{foo: bar}, {foo: baz}]\n", - }, - }, -} - -func TestFlattenOperatorScenarios(t *testing.T) { - for _, tt := range flattenOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "flatten", flattenOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_group_by.go b/external/yq/pkg/yqlib/operator_group_by.go deleted file mode 100755 index 3bb70b5..0000000 --- a/external/yq/pkg/yqlib/operator_group_by.go +++ /dev/null @@ -1,76 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "github.com/elliotchance/orderedmap" - yaml "gopkg.in/yaml.v3" -) - -func processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *yaml.Node) (*orderedmap.OrderedMap, error) { - var newMatches = orderedmap.NewOrderedMap() - for _, node := range node.Content { - child := &CandidateNode{Node: node} - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp) - - if err != nil { - return nil, err - } - - keyValue := "null" - - if rhs.MatchingNodes.Len() > 0 { - first := rhs.MatchingNodes.Front() - keyCandidate := first.Value.(*CandidateNode) - keyValue = keyCandidate.Node.Value - } - - groupList, exists := newMatches.Get(keyValue) - - if !exists { - groupList = list.New() - newMatches.Set(keyValue, groupList) - } - groupList.(*list.List).PushBack(node) - } - return newMatches, nil -} - -func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- groupBy Operator") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - - if candidateNode.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("Only arrays are supported for group by") - } - - newMatches, err := processIntoGroups(d, context, expressionNode.RHS, candidateNode) - - if err != nil { - return Context{}, err - } - - resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() { - groupResultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - groupList := groupEl.Value.(*list.List) - for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() { - groupResultNode.Content = append(groupResultNode.Content, groupItem.Value.(*yaml.Node)) - } - - resultNode.Content = append(resultNode.Content, groupResultNode) - } - - results.PushBack(candidate.CreateReplacement(resultNode)) - - } - - return context.ChildContext(results), nil - -} diff --git a/external/yq/pkg/yqlib/operator_group_by_test.go b/external/yq/pkg/yqlib/operator_group_by_test.go deleted file mode 100755 index 2c7803d..0000000 --- a/external/yq/pkg/yqlib/operator_group_by_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var groupByOperatorScenarios = []expressionScenario{ - { - description: "Group by field", - document: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`, - expression: `group_by(.foo)`, - expected: []string{ - "D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", - }, - }, - { - description: "Group by field, with nuls", - document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, - expression: `group_by(.foo)`, - expected: []string{ - "D0, P[], (!!seq)::- - {cat: dog}\n - {no: foo for you}\n- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", - }, - }, -} - -func TestGroupByOperatorScenarios(t *testing.T) { - for _, tt := range groupByOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "group-by", groupByOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_has.go b/external/yq/pkg/yqlib/operator_has.go deleted file mode 100755 index 01d7a1f..0000000 --- a/external/yq/pkg/yqlib/operator_has.go +++ /dev/null @@ -1,59 +0,0 @@ -package yqlib - -import ( - "container/list" - "strconv" - - yaml "gopkg.in/yaml.v3" -) - -func hasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- hasOperation") - var results = list.New() - - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - - wantedKey := "null" - wanted := &yaml.Node{Tag: "!!null"} - if rhs.MatchingNodes.Len() != 0 { - wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node - wantedKey = wanted.Value - } - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - // grab the first value - candidateNode := unwrapDoc(candidate.Node) - var contents = candidateNode.Content - switch candidateNode.Kind { - case yaml.MappingNode: - candidateHasKey := false - for index := 0; index < len(contents) && !candidateHasKey; index = index + 2 { - key := contents[index] - if key.Value == wantedKey { - candidateHasKey = true - } - } - results.PushBack(createBooleanCandidate(candidate, candidateHasKey)) - case yaml.SequenceNode: - candidateHasKey := false - if wanted.Tag == "!!int" { - var number, errParsingInt = strconv.ParseInt(wantedKey, 10, 64) - if errParsingInt != nil { - return Context{}, errParsingInt - } - candidateHasKey = int64(len(contents)) > number - } - results.PushBack(createBooleanCandidate(candidate, candidateHasKey)) - default: - results.PushBack(createBooleanCandidate(candidate, false)) - } - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_has_test.go b/external/yq/pkg/yqlib/operator_has_test.go deleted file mode 100755 index 3a63133..0000000 --- a/external/yq/pkg/yqlib/operator_has_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var hasOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `a: hello`, - expression: `has("a")`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `a: hello`, - expression: `has(.b) as $c`, - expected: []string{ - "D0, P[], (doc)::a: hello\n", - }, - }, - { - skipDoc: true, - document: `a: hello`, - expression: `has(.b)`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - description: "Has map key", - document: `- a: "yes" -- a: ~ -- a: -- b: nope -`, - expression: `.[] | has("a")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::true\n", - "D0, P[2], (!!bool)::true\n", - "D0, P[3], (!!bool)::false\n", - }, - }, - { - description: "Select, checking for existence of deep paths", - subdescription: "Simply pipe in parent expressions into `has`", - document: "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}", - expression: `.[] | select(.a.b | has("c"))`, - expected: []string{ - "D0, P[0], (!!map)::{a: {b: {c: cat}}}\n", - }, - }, - { - dontFormatInputForDoc: true, - description: "Has array index", - document: `- [] -- [1] -- [1, 2] -- [1, null] -- [1, 2, 3] -`, - expression: `.[] | has(1)`, - expected: []string{ - "D0, P[0], (!!bool)::false\n", - "D0, P[1], (!!bool)::false\n", - "D0, P[2], (!!bool)::true\n", - "D0, P[3], (!!bool)::true\n", - "D0, P[4], (!!bool)::true\n", - }, - }, -} - -func TestHasOperatorScenarios(t *testing.T) { - for _, tt := range hasOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "has", hasOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_keys.go b/external/yq/pkg/yqlib/operator_keys.go deleted file mode 100755 index 0e77310..0000000 --- a/external/yq/pkg/yqlib/operator_keys.go +++ /dev/null @@ -1,85 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "gopkg.in/yaml.v3" -) - -func isKeyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- isKeyOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - results.PushBack(createBooleanCandidate(candidate, candidate.IsMapKey)) - } - - return context.ChildContext(results), nil -} - -func getKeyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- getKeyOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - if candidate.Key != nil { - results.PushBack(candidate.CreateReplacement(candidate.Key)) - } - } - - return context.ChildContext(results), nil - -} - -func keysOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- keysOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - var targetNode *yaml.Node - if node.Kind == yaml.MappingNode { - targetNode = getMapKeys(node) - } else if node.Kind == yaml.SequenceNode { - targetNode = getIndicies(node) - } else { - return Context{}, fmt.Errorf("Cannot get keys of %v, keys only works for maps and arrays", node.Tag) - } - - result := candidate.CreateReplacement(targetNode) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} - -func getMapKeys(node *yaml.Node) *yaml.Node { - contents := make([]*yaml.Node, 0) - for index := 0; index < len(node.Content); index = index + 2 { - contents = append(contents, node.Content[index]) - } - return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents} -} - -func getIndicies(node *yaml.Node) *yaml.Node { - var contents = make([]*yaml.Node, len(node.Content)) - - for index := range node.Content { - contents[index] = &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!int", - Value: fmt.Sprintf("%v", index), - } - } - - return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents} -} diff --git a/external/yq/pkg/yqlib/operator_keys_test.go b/external/yq/pkg/yqlib/operator_keys_test.go deleted file mode 100755 index 3f821b5..0000000 --- a/external/yq/pkg/yqlib/operator_keys_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var expectedIsKey = `D0, P[], (!!seq)::- p: "" - isKey: false - tag: '!!map' -- p: a - isKey: true - tag: '!!str' -- p: a - isKey: false - tag: '!!map' -- p: a.b - isKey: true - tag: '!!str' -- p: a.b - isKey: false - tag: '!!seq' -- p: a.b.0 - isKey: false - tag: '!!str' -- p: a.c - isKey: true - tag: '!!str' -- p: a.c - isKey: false - tag: '!!str' -` - -var keysOperatorScenarios = []expressionScenario{ - { - description: "Map keys", - document: `{dog: woof, cat: meow}`, - expression: `keys`, - expected: []string{ - "D0, P[], (!!seq)::- dog\n- cat\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `keys`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - description: "Array keys", - document: `[apple, banana]`, - expression: `keys`, - expected: []string{ - "D0, P[], (!!seq)::- 0\n- 1\n", - }, - }, - { - skipDoc: true, - document: `[]`, - expression: `keys`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - description: "Retrieve array key", - document: "[1,2,3]", - expression: `.[1] | key`, - expected: []string{ - "D0, P[1], (!!int)::1\n", - }, - }, - { - description: "Retrieve map key", - document: "a: thing", - expression: `.a | key`, - expected: []string{ - "D0, P[a], (!!str)::a\n", - }, - }, - { - description: "No key", - document: "{}", - expression: `key`, - expected: []string{}, - }, - { - description: "Update map key", - document: "a:\n x: 3\n y: 4", - expression: `(.a.x | key) = "meow"`, - expected: []string{ - "D0, P[], (doc)::a:\n meow: 3\n y: 4\n", - }, - }, - { - description: "Get comment from map key", - document: "a: \n # comment on key\n x: 3\n y: 4", - expression: `.a.x | key | headComment`, - expected: []string{ - "D0, P[a x], (!!str)::comment on key\n", - }, - }, - { - description: "Check node is a key", - document: "a: \n b: [cat]\n c: frog\n", - expression: `[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]`, - expected: []string{ - expectedIsKey, - }, - }, -} - -func TestKeysOperatorScenarios(t *testing.T) { - for _, tt := range keysOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "keys", keysOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_length.go b/external/yq/pkg/yqlib/operator_length.go deleted file mode 100755 index 46f1c8c..0000000 --- a/external/yq/pkg/yqlib/operator_length.go +++ /dev/null @@ -1,39 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func lengthOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- lengthOperation") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - targetNode := unwrapDoc(candidate.Node) - var length int - switch targetNode.Kind { - case yaml.ScalarNode: - if targetNode.Tag == "!!null" { - length = 0 - } else { - length = len(targetNode.Value) - } - case yaml.MappingNode: - length = len(targetNode.Content) / 2 - case yaml.SequenceNode: - length = len(targetNode.Content) - default: - length = 0 - } - - node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_length_test.go b/external/yq/pkg/yqlib/operator_length_test.go deleted file mode 100755 index dee240a..0000000 --- a/external/yq/pkg/yqlib/operator_length_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var lengthOperatorScenarios = []expressionScenario{ - { - description: "String length", - subdescription: "returns length of string", - document: `{a: cat}`, - expression: `.a | length`, - expected: []string{ - "D0, P[a], (!!int)::3\n", - }, - }, - { - description: "null length", - document: `{a: null}`, - expression: `.a | length`, - expected: []string{ - "D0, P[a], (!!int)::0\n", - }, - }, - { - skipDoc: true, - document: `{a: ~}`, - expression: `.a | length`, - expected: []string{ - "D0, P[a], (!!int)::0\n", - }, - }, - { - skipDoc: true, - document: "# abc\n{a: key no exist}", - expression: `.b | length`, - expected: []string{ - "D0, P[b], (!!int)::0\n", - }, - }, - { - description: "Map length", - subdescription: "returns number of entries", - document: `{a: cat, c: dog}`, - expression: `length`, - expected: []string{ - "D0, P[], (!!int)::2\n", - }, - }, - { - description: "Array length", - subdescription: "returns number of elements", - document: `[2,4,6,8]`, - expression: `length`, - expected: []string{ - "D0, P[], (!!int)::4\n", - }, - }, -} - -func TestLengthOperatorScenarios(t *testing.T) { - for _, tt := range lengthOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "length", lengthOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_line.go b/external/yq/pkg/yqlib/operator_line.go deleted file mode 100755 index 1535e05..0000000 --- a/external/yq/pkg/yqlib/operator_line.go +++ /dev/null @@ -1,23 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func lineOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("lineOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.Node.Line), Tag: "!!int"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_line_test.go b/external/yq/pkg/yqlib/operator_line_test.go deleted file mode 100755 index e376524..0000000 --- a/external/yq/pkg/yqlib/operator_line_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var lineOperatorScenarios = []expressionScenario{ - { - description: "Returns line of _value_ node", - document: "a: cat\nb:\n c: cat", - expression: `.b | line`, - expected: []string{ - "D0, P[b], (!!int)::3\n", - }, - }, - { - description: "Returns line of _key_ node", - subdescription: "Pipe through the key operator to get the line of the key", - document: "a: cat\nb:\n c: cat", - expression: `.b | key| line`, - expected: []string{ - "D0, P[b], (!!int)::2\n", - }, - }, - { - description: "First line is 1", - document: "a: cat", - expression: `.a | line`, - expected: []string{ - "D0, P[a], (!!int)::1\n", - }, - }, - { - description: "No line data is 0", - expression: `{"a": "new entry"} | line`, - expected: []string{ - "D0, P[], (!!int)::0\n", - }, - }, -} - -func TestLineOperatorScenarios(t *testing.T) { - for _, tt := range lineOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "line", lineOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_load.go b/external/yq/pkg/yqlib/operator_load.go deleted file mode 100755 index eda63b7..0000000 --- a/external/yq/pkg/yqlib/operator_load.go +++ /dev/null @@ -1,97 +0,0 @@ -package yqlib - -import ( - "bufio" - "container/list" - "fmt" - "os" - - "gopkg.in/yaml.v3" -) - -type loadPrefs struct { - loadAsString bool - decoder Decoder -} - -func loadString(filename string) (*CandidateNode, error) { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside that directory. - - filebytes, err := os.ReadFile(filename) // #nosec - if err != nil { - return nil, err - } - - return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: string(filebytes)}}, nil -} - -func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) { - - file, err := os.Open(filename) // #nosec - if err != nil { - return nil, err - } - reader := bufio.NewReader(file) - - documents, err := readDocuments(reader, filename, 0, decoder) - if err != nil { - return nil, err - } - - if documents.Len() == 0 { - // return null candidate - return &CandidateNode{Node: &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}}, nil - } else if documents.Len() == 1 { - return documents.Front().Value.(*CandidateNode), nil - - } else { - sequenceNode := &CandidateNode{Node: &yaml.Node{Kind: yaml.SequenceNode}} - for doc := documents.Front(); doc != nil; doc = doc.Next() { - sequenceNode.Node.Content = append(sequenceNode.Node.Content, unwrapDoc(doc.Value.(*CandidateNode).Node)) - } - return sequenceNode, nil - } -} - -func loadYamlOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("loadYamlOperator") - - loadPrefs := expressionNode.Operation.Preferences.(loadPrefs) - - // need to evaluate the 1st parameter against the context - // and return the data accordingly. - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if rhs.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("Filename expression returned nil") - } - nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode) - - filename := nameCandidateNode.Node.Value - - var contentsCandidate *CandidateNode - - if loadPrefs.loadAsString { - contentsCandidate, err = loadString(filename) - } else { - contentsCandidate, err = loadYaml(filename, loadPrefs.decoder) - } - if err != nil { - return Context{}, fmt.Errorf("Failed to load %v: %w", filename, err) - } - - results.PushBack(contentsCandidate) - - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_load_test.go b/external/yq/pkg/yqlib/operator_load_test.go deleted file mode 100755 index 67a73e1..0000000 --- a/external/yq/pkg/yqlib/operator_load_test.go +++ /dev/null @@ -1,107 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var loadScenarios = []expressionScenario{ - { - skipDoc: true, - description: "Load empty file with a comment", - expression: `load("../../examples/empty.yaml")`, - expected: []string{ - "D0, P[], (doc)::# comment\n\n", - }, - }, - { - skipDoc: true, - description: "Load empty file with no comment", - expression: `load("../../examples/empty-no-comment.yaml")`, - expected: []string{ - "D0, P[], (!!null)::\n", - }, - }, - { - skipDoc: true, - description: "Load multiple documents", - expression: `load("../../examples/multiple_docs_small.yaml")`, - expected: []string{ - "D0, P[], ()::- a: Easy! as one two three\n- another:\n document: here\n- - 1\n - 2\n", - }, - }, - { - description: "Simple example", - document: `{myFile: "../../examples/thing.yml"}`, - expression: `load(.myFile)`, - expected: []string{ - "D0, P[], (doc)::a: apple is included\nb: cool.\n", - }, - }, - { - description: "Replace node with referenced file", - subdescription: "Note that you can modify the filename in the load operator if needed.", - document: `{something: {file: "thing.yml"}}`, - expression: `.something |= load("../../examples/" + .file)`, - expected: []string{ - "D0, P[], (doc)::{something: {a: apple is included, b: cool.}}\n", - }, - }, - { - description: "Replace _all_ nodes with referenced file", - subdescription: "Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. ", - document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`, - expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`, - expected: []string{ - "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n", - }, - }, - { - description: "Replace node with referenced file as string", - subdescription: "This will work for any text based file", - document: `{something: {file: "thing.yml"}}`, - expression: `.something |= load_str("../../examples/" + .file)`, - expected: []string{ - "D0, P[], (doc)::{something: \"a: apple is included\\nb: cool.\"}\n", - }, - }, - { - description: "Load from XML", - document: "cool: things", - expression: `.more_stuff = load_xml("../../examples/small.xml")`, - expected: []string{ - "D0, P[], (doc)::cool: things\nmore_stuff:\n this: is some xml\n", - }, - }, - { - description: "Load from Properties", - document: "cool: things", - expression: `.more_stuff = load_props("../../examples/small.properties")`, - expected: []string{ - "D0, P[], (doc)::cool: things\nmore_stuff:\n this:\n is: a properties file\n", - }, - }, - { - description: "Merge from properties", - subdescription: "This can be used as a convenient way to update a yaml document", - document: "this:\n is: from yaml\n cool: ay\n", - expression: `. *= load_props("../../examples/small.properties")`, - expected: []string{ - "D0, P[], (!!map)::this:\n is: a properties file\n cool: ay\n", - }, - }, - { - description: "Load from base64 encoded file", - document: "cool: things", - expression: `.more_stuff = load_base64("../../examples/base64.txt")`, - expected: []string{ - "D0, P[], (doc)::cool: things\nmore_stuff: my secret chilli recipe is....\n", - }, - }, -} - -func TestLoadScenarios(t *testing.T) { - for _, tt := range loadScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "load", loadScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_map.go b/external/yq/pkg/yqlib/operator_map.go deleted file mode 100755 index ec1a5ed..0000000 --- a/external/yq/pkg/yqlib/operator_map.go +++ /dev/null @@ -1,64 +0,0 @@ -package yqlib - -import ( - "container/list" -) - -func mapValuesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - //run expression against entries - // splat toEntries and pipe it into Rhs - splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) - if err != nil { - return Context{}, err - } - - assignUpdateExp := &ExpressionNode{ - Operation: &Operation{OperationType: assignOpType, UpdateAssign: true}, - RHS: expressionNode.RHS, - } - _, err = assignUpdateOperator(d, splatted, assignUpdateExp) - if err != nil { - return Context{}, err - } - - } - - return context, nil -} - -func mapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - //run expression against entries - // splat toEntries and pipe it into Rhs - splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) - if err != nil { - return Context{}, err - } - - result, err := d.GetMatchingNodes(splatted, expressionNode.RHS) - log.Debug("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType) - log.Debug("result %v", result) - if err != nil { - return Context{}, err - } - - selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} - collected, err := collectTogether(d, result, selfExpression) - if err != nil { - return Context{}, err - } - collected.Node.Style = unwrapDoc(candidate.Node).Style - - results.PushBack(collected) - - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_map_test.go b/external/yq/pkg/yqlib/operator_map_test.go deleted file mode 100755 index 715df33..0000000 --- a/external/yq/pkg/yqlib/operator_map_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var mapOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `[1,2,3]`, - document2: `[5,6,7]`, - expression: `map(. + 1)`, - expected: []string{ - "D0, P[], (!!seq)::[2, 3, 4]\n", - "D0, P[], (!!seq)::[6, 7, 8]\n", - }, - }, - { - description: "Map array", - document: `[1,2,3]`, - expression: `map(. + 1)`, - expected: []string{ - "D0, P[], (!!seq)::[2, 3, 4]\n", - }, - }, - { - skipDoc: true, - document: `{a: 1, b: 2, c: 3}`, - document2: `{x: 10, y: 20, z: 30}`, - expression: `map_values(. + 1)`, - expected: []string{ - "D0, P[], (doc)::{a: 2, b: 3, c: 4}\n", - "D0, P[], (doc)::{x: 11, y: 21, z: 31}\n", - }, - }, - { - description: "Map object values", - document: `{a: 1, b: 2, c: 3}`, - expression: `map_values(. + 1)`, - expected: []string{ - "D0, P[], (doc)::{a: 2, b: 3, c: 4}\n", - }, - }, -} - -func TestMapOperatorScenarios(t *testing.T) { - for _, tt := range mapOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "map", mapOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_multiply.go b/external/yq/pkg/yqlib/operator_multiply.go deleted file mode 100755 index ac27ad1..0000000 --- a/external/yq/pkg/yqlib/operator_multiply.go +++ /dev/null @@ -1,203 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "strconv" - "strings" - - "github.com/jinzhu/copier" - yaml "gopkg.in/yaml.v3" -) - -type multiplyPreferences struct { - AppendArrays bool - DeepMergeArrays bool - TraversePrefs traversePreferences - AssignPrefs assignPreferences -} - -func createMultiplyOp(prefs interface{}) func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { - return func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { - return &ExpressionNode{Operation: &Operation{OperationType: multiplyOpType, Preferences: prefs}, - LHS: lhs, - RHS: rhs} - } -} - -func multiplyAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var multiplyPrefs = expressionNode.Operation.Preferences - - return compoundAssignFunction(d, context, expressionNode, createMultiplyOp(multiplyPrefs)) -} - -func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- MultiplyOperator") - return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false) -} - -func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) { - leadingContent = rhs.LeadingContent - headComment = rhs.Node.HeadComment - footComment = rhs.Node.FootComment - if lhs.Node.HeadComment != "" || lhs.LeadingContent != "" { - headComment = lhs.Node.HeadComment - leadingContent = lhs.LeadingContent - } - - if lhs.Node.FootComment != "" { - footComment = lhs.Node.FootComment - } - return leadingContent, headComment, footComment -} - -func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - // need to do this before unWrapping the potential document node - leadingContent, headComment, footComment := getComments(lhs, rhs) - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - log.Debugf("Multiplying LHS: %v", lhs.Node.Tag) - log.Debugf("- RHS: %v", rhs.Node.Tag) - - if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || - (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { - var newBlank = CandidateNode{} - err := copier.CopyWithOption(&newBlank, lhs, copier.Option{IgnoreEmpty: true, DeepCopy: true}) - if err != nil { - return nil, err - } - newBlank.LeadingContent = leadingContent - newBlank.Node.HeadComment = headComment - newBlank.Node.FootComment = footComment - - return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences) - } - return multiplyScalars(lhs, rhs) - } -} - -func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - lhsTag := lhs.Node.Tag - rhsTag := guessTagFromCustomType(rhs.Node) - lhsIsCustom := false - if !strings.HasPrefix(lhsTag, "!!") { - // custom tag - we have to have a guess - lhsTag = guessTagFromCustomType(lhs.Node) - lhsIsCustom = true - } - - if lhsTag == "!!int" && rhsTag == "!!int" { - return multiplyIntegers(lhs, rhs) - } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { - return multiplyFloats(lhs, rhs, lhsIsCustom) - } - return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) -} - -func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode, lhsIsCustom bool) (*CandidateNode, error) { - target := lhs.CreateReplacement(&yaml.Node{}) - target.Node.Kind = yaml.ScalarNode - target.Node.Style = lhs.Node.Style - if lhsIsCustom { - target.Node.Tag = lhs.Node.Tag - } else { - target.Node.Tag = "!!float" - } - - lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64) - if err != nil { - return nil, err - } - rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64) - if err != nil { - return nil, err - } - target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum) - return target, nil -} - -func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - target := lhs.CreateReplacement(&yaml.Node{}) - target.Node.Kind = yaml.ScalarNode - target.Node.Style = lhs.Node.Style - target.Node.Tag = lhs.Node.Tag - - format, lhsNum, err := parseInt64(lhs.Node.Value) - if err != nil { - return nil, err - } - _, rhsNum, err := parseInt64(rhs.Node.Value) - if err != nil { - return nil, err - } - target.Node.Value = fmt.Sprintf(format, lhsNum*rhsNum) - return target, nil -} - -func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) { - var results = list.New() - - // only need to recurse the array if we are doing a deep merge - prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays, - TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}} - log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays) - log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays) - err := recursiveDecent(results, context.SingleChildContext(rhs), prefs) - if err != nil { - return nil, err - } - - var pathIndexToStartFrom int - if results.Front() != nil { - pathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).Path) - } - - for el := results.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - if candidate.Node.Tag == "!!merge" { - continue - } - - err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences) - if err != nil { - return nil, err - } - } - return lhs, nil -} - -func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error { - shouldAppendArrays := preferences.AppendArrays - log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey()) - - lhsPath := rhs.Path[pathIndexToStartFrom:] - log.Debugf("merge - lhsPath %v", lhsPath) - - assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs} - if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { - assignmentOp.OperationType = addAssignOpType - log.Debugf("merge - assignmentOp.OperationType = addAssignOpType") - } else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode || - (rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) { - assignmentOp.OperationType = assignOpType - assignmentOp.UpdateAssign = false - log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode) - log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode) - log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode) - log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign") - } else { - log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}") - } - rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs} - - assignmentOpNode := &ExpressionNode{ - Operation: assignmentOp, - LHS: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), - RHS: &ExpressionNode{Operation: rhsOp}, - } - - _, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode) - - return err -} diff --git a/external/yq/pkg/yqlib/operator_multiply_test.go b/external/yq/pkg/yqlib/operator_multiply_test.go deleted file mode 100755 index 4c9a206..0000000 --- a/external/yq/pkg/yqlib/operator_multiply_test.go +++ /dev/null @@ -1,589 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var doc1 = `list: -# Hi this is a comment. -# Hello this is another comment. -- "abc"` - -var doc2 = `list2: -# This is yet another comment. -# Indeed this is yet another comment. -- "123"` - -var docExpected = `D0, P[], (!!map)::list: - # Hi this is a comment. - # Hello this is another comment. - - "abc" -list2: - # This is yet another comment. - # Indeed this is yet another comment. - - "123" -` - -var mergeArrayWithAnchors = `sample: -- &a -- <<: *a -` - -var mergeArraysObjectKeysText = ` -This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below. - -It merges in the array provided in the second file into the first - matching on equal keys. - -Explanation: - -The approach, at a high level, is to reduce into a merged map (keyed by the unique key) -and then convert that back into an array. - -First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by. -The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. - -Next, we convert the map back to an array, using reduce again, concatenating all the map values together. - -Finally, we set the result of the merged array back into the first doc. - -Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) -` - -var mergeExpression = ` -( - (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap - | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) -) as $mergedArray -| select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray -` - -var docWithHeader = `# here - -a: apple -` - -var nodeWithHeader = `node: - # here - a: apple -` - -var docNoComments = `b: banana -` - -var docWithFooter = `a: apple - -# footer -` - -var nodeWithFooter = `a: apple -# footer` - -var document = `a: &cat {name: cat} -b: {name: dog} -c: - <<: *cat -` - -var multiplyOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: mergeArrayWithAnchors, - expression: `. * .`, - expected: []string{ - "D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n", - }, - }, - { - skipDoc: true, - document: `[[c], [b]]`, - expression: `.[] | . *+ ["a"]`, - expected: []string{ - "D0, P[0], (!!seq)::[c, a]\n", - "D0, P[1], (!!seq)::[b, a]\n", - }, - }, - { - skipDoc: true, - document: docWithHeader, - document2: docNoComments, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ - "D0, P[], (!!map)::# here\na: apple\nb: banana\n", - }, - }, - { - skipDoc: true, - document: nodeWithHeader, - document2: docNoComments, - expression: `(select(fi == 0) | .node) * select(fi == 1)`, - expected: []string{ - "D0, P[node], (!!map)::# here\na: apple\nb: banana\n", - }, - }, - { - skipDoc: true, - document: docNoComments, - document2: docWithHeader, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ - "D0, P[], (!!map)::# here\nb: banana\na: apple\n", - }, - }, - { - skipDoc: true, - document: docNoComments, - document2: nodeWithHeader, - expression: `select(fi == 0) * (select(fi == 1) | .node)`, - expected: []string{ - "D0, P[], (!!map)::b: banana\n# here\na: apple\n", - }, - }, - { - skipDoc: true, - document: docWithFooter, - document2: docNoComments, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ - "D0, P[], (!!map)::a: apple\nb: banana\n# footer\n", - }, - }, - { - skipDoc: true, - document: nodeWithFooter, - document2: docNoComments, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ // not sure why there's an extra newline *shrug* - "D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n", - }, - }, - { - description: "doc 2 has footer", - skipDoc: true, - document: docNoComments, - document2: docWithFooter, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ - "D0, P[], (!!map)::b: banana\na: apple\n# footer\n", - }, - }, - { - description: "Multiply integers", - document: "a: 3\nb: 4", - expression: `.a *= .b`, - expected: []string{ - "D0, P[], (doc)::a: 12\nb: 4\n", - }, - }, - { - skipDoc: true, - document: doc1, - document2: doc2, - expression: `select(fi == 0) * select(fi == 1)`, - expected: []string{ - docExpected, - }, - }, - { - skipDoc: true, - expression: `.x = {"things": "whatever"} * {}`, - expected: []string{ - "D0, P[], ()::x:\n things: whatever\n", - }, - }, - { - skipDoc: true, - expression: `.x = {} * {"things": "whatever"}`, - expected: []string{ - "D0, P[], ()::x:\n things: whatever\n", - }, - }, - { - skipDoc: true, - expression: `3 * 4.5`, - expected: []string{ - "D0, P[], (!!float)::13.5\n", - }, - }, - { - skipDoc: true, - expression: `4.5 * 3`, - expected: []string{ - "D0, P[], (!!float)::13.5\n", - }, - }, - { - skipDoc: true, - document: `{a: {also: [1]}, b: {also: me}}`, - expression: `. * {"a" : .b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", - }, - }, - { - skipDoc: true, - document: "# b\nb:\n # a\n a: cat", - expression: "{} * .", - expected: []string{ - "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", - }, - }, - { - skipDoc: true, - document: "# b\nb:\n # a\n a: cat", - expression: ". * {}", - expected: []string{ - "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", - }, - }, - { - skipDoc: true, - document: `{a: &a { b: &b { c: &c cat } } }`, - expression: `{} * .`, - expected: []string{ - "D0, P[], (!!map)::a: &a\n b: &b\n c: &c cat\n", - }, - }, - { - skipDoc: true, - document: `{a: 2, b: 5}`, - document2: `{a: 3, b: 10}`, - expression: `.a * .b`, - expected: []string{ - "D0, P[a], (!!int)::10\n", - "D0, P[a], (!!int)::20\n", - "D0, P[a], (!!int)::15\n", - "D0, P[a], (!!int)::30\n", - }, - }, - { - skipDoc: true, - document: `{a: 2}`, - document2: `{b: 10}`, - expression: `select(fi ==0) * select(fi==1)`, - expected: []string{ - "D0, P[], (!!map)::{a: 2, b: 10}\n", - }, - }, - { - skipDoc: true, - expression: `{} * {"cat":"dog"}`, - expected: []string{ - "D0, P[], (!!map)::cat: dog\n", - }, - }, - { - skipDoc: true, - document: `{a: {also: me}, b: {also: [1]}}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", - }, - }, - { - description: "Merge objects together, returning merged result only", - document: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`, - expression: `.a * .b`, - expected: []string{ - "D0, P[a], (!!map)::{field: {g: wizz}, fieldA: cat, fieldB: dog}\n", - }, - }, - { - description: "Merge objects together, returning parent object", - document: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {field: {g: wizz}, fieldA: cat, fieldB: dog}, b: {field: {g: wizz}, fieldB: dog}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {also: {g: wizz}}, b: {also: me}}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {also: {g: wizz}}, b: {also: [1]}}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {also: [1]}, b: {also: {g: wizz}}}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", - }, - }, - { - skipDoc: true, - document: `{a: {things: great}, b: {also: me}}`, - expression: `. * {"a": .b}`, - expected: []string{ - "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", - }, - }, - { - description: "Merge keeps style of LHS", - dontFormatInputForDoc: true, - document: "a: {things: great}\nb:\n also: \"me\"", - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::a: {things: great, also: \"me\"}\nb:\n also: \"me\"\n", - }, - }, - { - description: "Merge arrays", - document: `{a: [1,2,3], b: [3,4,5]}`, - expression: `. * {"a":.b}`, - expected: []string{ - "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n", - }, - }, - { - skipDoc: true, - document: `{a: [1], b: [2]}`, - expression: `.a *+ .b`, - expected: []string{ - "D0, P[a], (!!seq)::[1, 2]\n", - }, - }, - { - description: "Merge, only existing fields", - document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, - expression: `.a *? .b`, - expected: []string{ - "D0, P[a], (!!map)::{thing: two, cat: frog}\n", - }, - }, - { - description: "Merge, only new fields", - document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, - expression: `.a *n .b`, - expected: []string{ - "D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n", - }, - }, - { - skipDoc: true, - document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, - expression: `.a *?d .b`, - expected: []string{ - "D0, P[a], (!!seq)::[{thing: two}]\n", - }, - }, - { - skipDoc: true, - document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, - expression: `.a *nd .b`, - expected: []string{ - "D0, P[a], (!!seq)::[{thing: one, missing: two}]\n", - }, - }, - { - skipDoc: true, - document: `{a: {array: [1]}, b: {}}`, - expression: `.b *+ .a`, - expected: []string{ - "D0, P[b], (!!map)::array: [1]\n", - }, - }, - { - description: "Merge, appending arrays", - document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`, - expression: `.a *+ .b`, - expected: []string{ - "D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n", - }, - }, - { - description: "Merge, only existing fields, appending arrays", - document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, - expression: `.a *?+ .b`, - expected: []string{ - "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n", - }, - }, - { - description: "Merge, only new fields, appending arrays", - subdescription: "Append (+) with (n) has no effect.", - skipDoc: true, - document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, - expression: `.a *n+ .b`, - expected: []string{ - "D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n", - }, - }, - { - description: "Merge, deeply merging arrays", - subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.", - document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`, - expression: `.a *d .b`, - expected: []string{ - "D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n", - }, - }, - { - description: "Merge arrays of objects together, matching on a key", - subdescription: mergeArraysObjectKeysText, - document: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`, - document2: `newArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`, - environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"}, - expression: mergeExpression, - expected: []string{ - "D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n", - }, - }, - { - description: "Merge to prefix an element", - document: `{a: cat, b: dog}`, - expression: `. * {"a": {"c": .a}}`, - expected: []string{ - "D0, P[], (!!map)::{a: {c: cat}, b: dog}\n", - }, - }, - { - description: "Merge with simple aliases", - document: `{a: &cat {c: frog}, b: {f: *cat}, c: {g: thongs}}`, - expression: `.c * .b`, - expected: []string{ - "D0, P[c], (!!map)::{g: thongs, f: *cat}\n", - }, - }, - { - description: "Merge copies anchor names", - document: `{a: {c: &cat frog}, b: {f: *cat}, c: {g: thongs}}`, - expression: `.c * .a`, - expected: []string{ - "D0, P[c], (!!map)::{g: thongs, c: &cat frog}\n", - }, - }, - { - description: "Merge with merge anchors", - document: mergeDocSample, - expression: `.foobar * .foobarList`, - expected: []string{ - "D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n", - }, - }, - { - skipDoc: true, - document: document, - expression: `.b * .c`, - expected: []string{ - "D0, P[b], (!!map)::{name: dog, <<: *cat}\n", - }, - }, - { - description: "Custom types: that are really numbers", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse 2\nb: !goat 3", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: !horse 6\nb: !goat 3\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really numbers", - document: "a: !horse 2.5\nb: !goat 3.5", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: !horse 8.75\nb: !goat 3.5\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really numbers", - document: "a: 2\nb: !goat 3.5", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: !!float 7\nb: !goat 3.5\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really arrays", - document: "a: !horse [1,2]\nb: !goat [3]", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: !horse [3]\nb: !goat [3]\n", - }, - }, - { - description: "Custom types: that are really maps", - subdescription: "Custom tags will be maintained.", - document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", - }, - }, - { - description: "Custom types: clobber tags", - subdescription: "Use the `c` option to clobber custom tags. Note that the second tag is now used", - document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", - expression: ".a *=c .b", - expected: []string{ - "D0, P[], (doc)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", - }, - }, - { - skipDoc: true, - description: "Custom types: clobber tags - *=", - subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used", - document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", - expression: ".a =c .a *c .b", - expected: []string{ - "D0, P[], (doc)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", - }, - }, - { - skipDoc: true, - description: "Custom types: dont clobber tags - *=", - subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used", - document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", - expression: ".a *= .b", - expected: []string{ - "D0, P[], (doc)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really maps", - document: "a: {cat: !horse meow}\nb: {cat: 5}", - expression: ".a = .a * .b", - expected: []string{ - "D0, P[], (doc)::a: {cat: !horse 5}\nb: {cat: 5}\n", - }, - }, - { - skipDoc: true, - description: "Relative merge, new fields only", - document: "a: {a: original}\n", - expression: `.a *=n load("../../examples/thing.yml")`, - expected: []string{ - "D0, P[], (doc)::a: {a: original, b: cool.}\n", - }, - }, - { - skipDoc: true, - description: "Relative merge", - document: "a: {a: original}\n", - expression: `.a *= load("../../examples/thing.yml")`, - expected: []string{ - "D0, P[], (doc)::a: {a: apple is included, b: cool.}\n", - }, - }, -} - -func TestMultiplyOperatorScenarios(t *testing.T) { - for _, tt := range multiplyOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "multiply-merge", multiplyOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_parent.go b/external/yq/pkg/yqlib/operator_parent.go deleted file mode 100755 index ab0f03d..0000000 --- a/external/yq/pkg/yqlib/operator_parent.go +++ /dev/null @@ -1,19 +0,0 @@ -package yqlib - -import "container/list" - -func getParentOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- getParentOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - if candidate.Parent != nil { - results.PushBack(candidate.Parent) - } - } - - return context.ChildContext(results), nil - -} diff --git a/external/yq/pkg/yqlib/operator_parent_test.go b/external/yq/pkg/yqlib/operator_parent_test.go deleted file mode 100755 index 35afd67..0000000 --- a/external/yq/pkg/yqlib/operator_parent_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var parentOperatorScenarios = []expressionScenario{ - { - description: "Simple example", - document: `a: {nested: cat}`, - expression: `.a.nested | parent`, - expected: []string{ - "D0, P[a], (!!map)::{nested: cat}\n", - }, - }, - { - description: "Parent of nested matches", - document: `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`, - expression: `.. | select(. == "banana") | parent`, - expected: []string{ - "D0, P[b], (!!map)::{fruit: banana, name: sam}\n", - }, - }, - { - description: "No parent", - document: `{}`, - expression: `parent`, - expected: []string{}, - }, -} - -func TestParentOperatorScenarios(t *testing.T) { - for _, tt := range parentOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "parent", parentOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_path.go b/external/yq/pkg/yqlib/operator_path.go deleted file mode 100755 index 3905d37..0000000 --- a/external/yq/pkg/yqlib/operator_path.go +++ /dev/null @@ -1,172 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func createPathNodeFor(pathElement interface{}) *yaml.Node { - switch pathElement := pathElement.(type) { - case string: - return &yaml.Node{Kind: yaml.ScalarNode, Value: pathElement, Tag: "!!str"} - default: - return &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", pathElement), Tag: "!!int"} - } -} - -func getPathArrayFromNode(funcName string, node *yaml.Node) ([]interface{}, error) { - if node.Kind != yaml.SequenceNode { - return nil, fmt.Errorf("%v: expected path array, but got %v instead", funcName, node.Tag) - } - - path := make([]interface{}, len(node.Content)) - - for i, childNode := range node.Content { - if childNode.Tag == "!!str" { - path[i] = childNode.Value - } else if childNode.Tag == "!!int" { - number, err := parseInt(childNode.Value) - if err != nil { - return nil, fmt.Errorf("%v: could not parse %v as an int: %w", funcName, childNode.Value, err) - } - path[i] = number - } else { - return nil, fmt.Errorf("%v: expected either a !!str or !!int in the path, found %v instead", funcName, childNode.Tag) - } - - } - return path, nil -} - -// SETPATH(pathArray; value) -func setPathOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("SetPath") - - if expressionNode.RHS.Operation.OperationType != blockOpType { - return Context{}, fmt.Errorf("SETPATH must be given a block (;), got %v instead", expressionNode.RHS.Operation.OperationType.Type) - } - - lhsPathContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS.LHS) - - if err != nil { - return Context{}, err - } - - if lhsPathContext.MatchingNodes.Len() != 1 { - return Context{}, fmt.Errorf("SETPATH: expected single path but found %v results instead", lhsPathContext.MatchingNodes.Len()) - } - lhsValue := lhsPathContext.MatchingNodes.Front().Value.(*CandidateNode) - - lhsPath, err := getPathArrayFromNode("SETPATH", lhsValue.Node) - - if err != nil { - return Context{}, err - } - - lhsTraversalTree := createTraversalTree(lhsPath, traversePreferences{}, false) - - assignmentOp := &Operation{OperationType: assignOpType} - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - targetContextValue, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS.RHS) - if err != nil { - return Context{}, err - } - - if targetContextValue.MatchingNodes.Len() != 1 { - return Context{}, fmt.Errorf("SETPATH: expected single value on RHS but found %v", targetContextValue.MatchingNodes.Len()) - } - - rhsOp := &Operation{OperationType: valueOpType, CandidateNode: targetContextValue.MatchingNodes.Front().Value.(*CandidateNode)} - - assignmentOpNode := &ExpressionNode{ - Operation: assignmentOp, - LHS: lhsTraversalTree, - RHS: &ExpressionNode{Operation: rhsOp}, - } - - _, err = d.GetMatchingNodes(context.SingleChildContext(candidate), assignmentOpNode) - - if err != nil { - return Context{}, err - } - - } - return context, nil -} - -func delPathsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("delPaths") - // single RHS expression that returns an array of paths (array of arrays) - - pathArraysContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if pathArraysContext.MatchingNodes.Len() != 1 { - return Context{}, fmt.Errorf("DELPATHS: expected single value but found %v", pathArraysContext.MatchingNodes.Len()) - } - pathArraysNode := pathArraysContext.MatchingNodes.Front().Value.(*CandidateNode).Node - - if pathArraysNode.Tag != "!!seq" { - return Context{}, fmt.Errorf("DELPATHS: expected a sequence of sequences, but found %v", pathArraysNode.Tag) - } - - updatedContext := context - - for i, child := range pathArraysNode.Content { - - if child.Tag != "!!seq" { - return Context{}, fmt.Errorf("DELPATHS: expected entry [%v] to be a sequence, but its a %v. Note that delpaths takes an array of path arrays, e.g. [[\"a\", \"b\"]]", i, child.Tag) - } - childPath, err := getPathArrayFromNode("DELPATHS", child) - - if err != nil { - return Context{}, err - } - - childTraversalExp := createTraversalTree(childPath, traversePreferences{}, false) - deleteChildOp := &Operation{OperationType: deleteChildOpType} - - deleteChildOpNode := &ExpressionNode{ - Operation: deleteChildOp, - RHS: childTraversalExp, - } - - updatedContext, err = d.GetMatchingNodes(updatedContext, deleteChildOpNode) - - if err != nil { - return Context{}, err - } - - } - - return updatedContext, nil - -} - -func getPathOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetPath") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - - content := make([]*yaml.Node, len(candidate.Path)) - for pathIndex := 0; pathIndex < len(candidate.Path); pathIndex++ { - path := candidate.Path[pathIndex] - content[pathIndex] = createPathNodeFor(path) - } - node.Content = content - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_path_test.go b/external/yq/pkg/yqlib/operator_path_test.go deleted file mode 100755 index 084ba9d..0000000 --- a/external/yq/pkg/yqlib/operator_path_test.go +++ /dev/null @@ -1,146 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var documentToPrune = ` -parentA: bob -parentB: - child1: i am child1 - child2: i am child2 -parentC: - child1: me child1 - child2: me child2 -` - -var pathOperatorScenarios = []expressionScenario{ - { - description: "Map path", - document: `{a: {b: cat}}`, - expression: `.a.b | path`, - expected: []string{ - "D0, P[a b], (!!seq)::- a\n- b\n", - }, - }, - { - skipDoc: true, - document: `a: - b: - c: - - 0 - - 1 - - 2 - - 3`, - expression: `.a.b.c.[]`, - expected: []string{ - "D0, P[a b c 0], (!!int)::0\n", - "D0, P[a b c 1], (!!int)::1\n", - "D0, P[a b c 2], (!!int)::2\n", - "D0, P[a b c 3], (!!int)::3\n", - }, - }, - { - description: "Get map key", - document: `{a: {b: cat}}`, - expression: `.a.b | path | .[-1]`, - expected: []string{ - "D0, P[a b -1], (!!str)::b\n", - }, - }, - { - description: "Array path", - document: `{a: [cat, dog]}`, - expression: `.a.[] | select(. == "dog") | path`, - expected: []string{ - "D0, P[a 1], (!!seq)::- a\n- 1\n", - }, - }, - { - description: "Get array index", - document: `{a: [cat, dog]}`, - expression: `.a.[] | select(. == "dog") | path | .[-1]`, - expected: []string{ - "D0, P[a 1 -1], (!!int)::1\n", - }, - }, - { - description: "Print path and value", - document: `{a: [cat, dog, frog]}`, - expression: `.a[] | select(. == "*og") | [{"path":path, "value":.}]`, - expected: []string{ - "D0, P[a 1], (!!seq)::- path:\n - a\n - 1\n value: dog\n", - "D0, P[a 2], (!!seq)::- path:\n - a\n - 2\n value: frog\n", - }, - }, - { - description: "Set path", - document: `{a: {b: cat}}`, - expression: `setpath(["a", "b"]; "things")`, - expected: []string{ - "D0, P[], (doc)::{a: {b: things}}\n", - }, - }, - { - description: "Set on empty document", - expression: `setpath(["a", "b"]; "things")`, - expected: []string{ - "D0, P[], ()::a:\n b: things\n", - }, - }, - { - description: "Set path to prune deep paths", - subdescription: "Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object,", - document: documentToPrune, - expression: "(.parentB.child2, .parentC.child1) as $i\n ireduce({}; setpath($i | path; $i))", - expected: []string{ - "D0, P[], (!!map)::parentB:\n child2: i am child2\nparentC:\n child1: me child1\n", - }, - }, - { - description: "Set array path", - document: `a: [cat, frog]`, - expression: `setpath(["a", 0]; "things")`, - expected: []string{ - "D0, P[], (doc)::a: [things, frog]\n", - }, - }, - { - description: "Set array path empty", - expression: `setpath(["a", 0]; "things")`, - expected: []string{ - "D0, P[], ()::a:\n - things\n", - }, - }, - { - description: "Delete path", - subdescription: "Notice delpaths takes an _array_ of paths.", - document: `{a: {b: cat, c: dog, d: frog}}`, - expression: `delpaths([["a", "c"], ["a", "d"]])`, - expected: []string{ - "D0, P[], (doc)::{a: {b: cat}}\n", - }, - }, - { - description: "Delete array path", - document: `a: [cat, frog]`, - expression: `delpaths([["a", 0]])`, - expected: []string{ - "D0, P[], (doc)::a: [frog]\n", - }, - }, - { - description: "Delete - wrong parameter", - subdescription: "delpaths does not work with a single path array", - document: `a: [cat, frog]`, - expression: `delpaths(["a", 0])`, - expectedError: "DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [[\"a\", \"b\"]]", - }, -} - -func TestPathOperatorsScenarios(t *testing.T) { - for _, tt := range pathOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "path", pathOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_pick.go b/external/yq/pkg/yqlib/operator_pick.go deleted file mode 100755 index 416d947..0000000 --- a/external/yq/pkg/yqlib/operator_pick.go +++ /dev/null @@ -1,86 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func pickMap(original *yaml.Node, indices *yaml.Node) *yaml.Node { - - filteredContent := make([]*yaml.Node, 0) - for index := 0; index < len(indices.Content); index = index + 1 { - keyToFind := indices.Content[index] - - indexInMap := findKeyInMap(original, keyToFind) - if indexInMap > -1 { - clonedKey := deepClone(original.Content[indexInMap]) - clonedValue := deepClone(original.Content[indexInMap+1]) - filteredContent = append(filteredContent, clonedKey, clonedValue) - } - } - - newNode := deepCloneNoContent(original) - newNode.Content = filteredContent - - return newNode -} - -func pickSequence(original *yaml.Node, indices *yaml.Node) (*yaml.Node, error) { - - filteredContent := make([]*yaml.Node, 0) - for index := 0; index < len(indices.Content); index = index + 1 { - indexInArray, err := parseInt(indices.Content[index].Value) - if err != nil { - return nil, fmt.Errorf("cannot index array with %v", indices.Content[index].Value) - } - - if indexInArray > -1 && indexInArray < len(original.Content) { - filteredContent = append(filteredContent, deepClone(original.Content[indexInArray])) - } - } - - newNode := deepCloneNoContent(original) - newNode.Content = filteredContent - - return newNode, nil -} - -func pickOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("Pick") - - contextIndicesToPick, err := d.GetMatchingNodes(context, expressionNode.RHS) - - if err != nil { - return Context{}, err - } - indicesToPick := &yaml.Node{} - if contextIndicesToPick.MatchingNodes.Len() > 0 { - indicesToPick = contextIndicesToPick.MatchingNodes.Front().Value.(*CandidateNode).Node - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - - var replacement *yaml.Node - if node.Kind == yaml.MappingNode { - replacement = pickMap(node, indicesToPick) - } else if node.Kind == yaml.SequenceNode { - replacement, err = pickSequence(node, indicesToPick) - if err != nil { - return Context{}, err - } - - } else { - return Context{}, fmt.Errorf("cannot pick indicies from type %v (%v)", node.Tag, candidate.GetNicePath()) - } - - results.PushBack(candidate.CreateReplacementWithDocWrappers(replacement)) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_pick_test.go b/external/yq/pkg/yqlib/operator_pick_test.go deleted file mode 100755 index 52fe9e2..0000000 --- a/external/yq/pkg/yqlib/operator_pick_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var pickOperatorScenarios = []expressionScenario{ - { - description: "Pick keys from map", - subdescription: "Note that the order of the keys matches the pick order and non existent keys are skipped.", - document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n", - expression: `.myMap |= pick(["hamster", "cat", "goat"])`, - expected: []string{ - "D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n", - }, - }, - { - description: "Pick keys from map", - skipDoc: true, - document: "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n", - expression: `.myMap |= pick(["hamster", "cat", "goat"])`, - expected: []string{ - "D0, P[], (doc)::!things myMap: {hamster: squeek, cat: meow}\n", - }, - }, - { - description: "Pick keys from map with comments", - skipDoc: true, - document: "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n# xyz\n", - expression: `.myMap |= pick(["hamster", "cat", "goat"])`, - expected: []string{ - "D0, P[], (doc)::# abc\nmyMap: {hamster: squeek, cat: meow}\n# xyz\n", - }, - }, - { - description: "Pick indices from array", - subdescription: "Note that the order of the indexes matches the pick order and non existent indexes are skipped.", - document: `[cat, leopard, lion]`, - expression: `pick([2, 0, 734, -5])`, - expected: []string{ - "D0, P[], (!!seq)::[lion, cat]\n", - }, - }, - { - description: "Pick indices from array with comments", - skipDoc: true, - document: "# abc\n[cat, leopard, lion]\n# xyz", - expression: `pick([2, 0, 734, -5])`, - expected: []string{ - "D0, P[], (!!seq)::# abc\n[lion, cat]\n# xyz\n", - }, - }, -} - -func TestPickOperatorScenarios(t *testing.T) { - for _, tt := range pickOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "pick", pickOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_pipe.go b/external/yq/pkg/yqlib/operator_pipe.go deleted file mode 100755 index 77813c6..0000000 --- a/external/yq/pkg/yqlib/operator_pipe.go +++ /dev/null @@ -1,18 +0,0 @@ -package yqlib - -func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - //lhs may update the variable context, we should pass that into the RHS - // BUT we still return the original context back (see jq) - // https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|... - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - rhs, err := d.GetMatchingNodes(lhs, expressionNode.RHS) - if err != nil { - return Context{}, err - } - return context.ChildContext(rhs.MatchingNodes), nil -} diff --git a/external/yq/pkg/yqlib/operator_pipe_test.go b/external/yq/pkg/yqlib/operator_pipe_test.go deleted file mode 100755 index 9e97960..0000000 --- a/external/yq/pkg/yqlib/operator_pipe_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var pipeOperatorScenarios = []expressionScenario{ - { - description: "Simple Pipe", - document: `{a: {b: cat}}`, - expression: `.a | .b`, - expected: []string{ - "D0, P[a b], (!!str)::cat\n", - }, - }, - { - description: "Multiple updates", - document: `{a: cow, b: sheep, c: same}`, - expression: `.a = "cat" | .b = "dog"`, - expected: []string{ - "D0, P[], (doc)::{a: cat, b: dog, c: same}\n", - }, - }, -} - -func TestPipeOperatorScenarios(t *testing.T) { - for _, tt := range pipeOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "pipe", pipeOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_recursive_descent.go b/external/yq/pkg/yqlib/operator_recursive_descent.go deleted file mode 100755 index bb58384..0000000 --- a/external/yq/pkg/yqlib/operator_recursive_descent.go +++ /dev/null @@ -1,50 +0,0 @@ -package yqlib - -import ( - "container/list" - - yaml "gopkg.in/yaml.v3" -) - -type recursiveDescentPreferences struct { - TraversePreferences traversePreferences - RecurseArray bool -} - -func recursiveDescentOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - var results = list.New() - - preferences := expressionNode.Operation.Preferences.(recursiveDescentPreferences) - err := recursiveDecent(results, context, preferences) - if err != nil { - return Context{}, err - } - - return context.ChildContext(results), nil -} - -func recursiveDecent(results *list.List, context Context, preferences recursiveDescentPreferences) error { - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - candidate.Node = unwrapDoc(candidate.Node) - - log.Debugf("Recursive Decent, added %v", NodeToString(candidate)) - results.PushBack(candidate) - - if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 && - (preferences.RecurseArray || candidate.Node.Kind != yaml.SequenceNode) { - - children, err := splat(context.SingleChildContext(candidate), preferences.TraversePreferences) - - if err != nil { - return err - } - err = recursiveDecent(results, children, preferences) - if err != nil { - return err - } - } - } - return nil -} diff --git a/external/yq/pkg/yqlib/operator_recursive_descent_test.go b/external/yq/pkg/yqlib/operator_recursive_descent_test.go deleted file mode 100755 index 45e63cf..0000000 --- a/external/yq/pkg/yqlib/operator_recursive_descent_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var recursiveDescentOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `{}`, - expression: `..`, - expected: []string{ - "D0, P[], (!!map)::{}\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `...`, - expected: []string{ - "D0, P[], (!!map)::{}\n", - }, - }, - { - skipDoc: true, - document: `[]`, - expression: `..`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - skipDoc: true, - document: `[]`, - expression: `...`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - skipDoc: true, - document: `cat`, - expression: `..`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - skipDoc: true, - document: `cat`, - expression: `...`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - description: "Recurse map (values only)", - document: `{a: frog}`, - expression: `..`, - expected: []string{ - "D0, P[], (!!map)::{a: frog}\n", - "D0, P[a], (!!str)::frog\n", - }, - }, - { - description: "Recursively find nodes with keys", - subdescription: "Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.", - document: `{a: {name: frog, b: {name: blog, age: 12}}}`, - expression: `[.. | select(has("name"))]`, - expected: []string{ - "D0, P[], (!!seq)::- {name: frog, b: {name: blog, age: 12}}\n- {name: blog, age: 12}\n", - }, - }, - { - description: "Recursively find nodes with values", - document: `{a: {nameA: frog, b: {nameB: frog, age: 12}}}`, - expression: `.. | select(. == "frog")`, - expected: []string{ - "D0, P[a nameA], (!!str)::frog\n", - "D0, P[a b nameB], (!!str)::frog\n", - }, - }, - { - description: "Recurse map (values and keys)", - subdescription: "Note that the map key appears in the results", - document: `{a: frog}`, - expression: `...`, - expected: []string{ - "D0, P[], (!!map)::{a: frog}\n", - "D0, P[a], (!!str)::a\n", - "D0, P[a], (!!str)::frog\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple}}`, - expression: `..`, - expected: []string{ - "D0, P[], (!!map)::{a: {b: apple}}\n", - "D0, P[a], (!!map)::{b: apple}\n", - "D0, P[a b], (!!str)::apple\n", - }, - }, - { - skipDoc: true, - document: `{a: {b: apple}}`, - expression: `...`, - expected: []string{ - "D0, P[], (!!map)::{a: {b: apple}}\n", - "D0, P[a], (!!str)::a\n", - "D0, P[a], (!!map)::{b: apple}\n", - "D0, P[a b], (!!str)::b\n", - "D0, P[a b], (!!str)::apple\n", - }, - }, - { - skipDoc: true, - document: `[1,2,3]`, - expression: `..`, - expected: []string{ - "D0, P[], (!!seq)::[1, 2, 3]\n", - "D0, P[0], (!!int)::1\n", - "D0, P[1], (!!int)::2\n", - "D0, P[2], (!!int)::3\n", - }, - }, - { - skipDoc: true, - document: `[1,2,3]`, - expression: `...`, - expected: []string{ - "D0, P[], (!!seq)::[1, 2, 3]\n", - "D0, P[0], (!!int)::1\n", - "D0, P[1], (!!int)::2\n", - "D0, P[2], (!!int)::3\n", - }, - }, - { - skipDoc: true, - document: `[{a: cat},2,true]`, - expression: `..`, - expected: []string{ - "D0, P[], (!!seq)::[{a: cat}, 2, true]\n", - "D0, P[0], (!!map)::{a: cat}\n", - "D0, P[0 a], (!!str)::cat\n", - "D0, P[1], (!!int)::2\n", - "D0, P[2], (!!bool)::true\n", - }, - }, - { - skipDoc: true, - document: `[{a: cat},2,true]`, - expression: `...`, - expected: []string{ - "D0, P[], (!!seq)::[{a: cat}, 2, true]\n", - "D0, P[0], (!!map)::{a: cat}\n", - "D0, P[0 a], (!!str)::a\n", - "D0, P[0 a], (!!str)::cat\n", - "D0, P[1], (!!int)::2\n", - "D0, P[2], (!!bool)::true\n", - }, - }, - { - description: "Aliases are not traversed", - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `[..]`, - expected: []string{ - "D0, P[], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n", - }, - }, - { - skipDoc: true, - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `...`, - expected: []string{ - "D0, P[], (!!map)::{a: &cat {c: frog}, b: *cat}\n", - "D0, P[a], (!!str)::a\n", - "D0, P[a], (!!map)::&cat {c: frog}\n", - "D0, P[a c], (!!str)::c\n", - "D0, P[a c], (!!str)::frog\n", - "D0, P[b], (!!str)::b\n", - "D0, P[b], (alias)::*cat\n", - }, - }, - { - description: "Merge docs are not traversed", - document: mergeDocSample, - expression: `.foobar | [..]`, - expected: []string{ - "D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobar | [...]`, - expected: []string{ - "D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList | ..`, - expected: []string{ - "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", - "D0, P[foobarList b], (!!str)::foobarList_b\n", - "D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n", - "D0, P[foobarList << 0], (alias)::*foo\n", - "D0, P[foobarList << 1], (alias)::*bar\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList | ...`, - expected: []string{ - "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", - "D0, P[foobarList b], (!!str)::b\n", - "D0, P[foobarList b], (!!str)::foobarList_b\n", - "D0, P[foobarList <<], (!!merge)::<<\n", - "D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n", - "D0, P[foobarList << 0], (alias)::*foo\n", - "D0, P[foobarList << 1], (alias)::*bar\n", - "D0, P[foobarList c], (!!str)::c\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, -} - -func TestRecursiveDescentOperatorScenarios(t *testing.T) { - for _, tt := range recursiveDescentOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "recursive-descent-glob", recursiveDescentOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_reduce.go b/external/yq/pkg/yqlib/operator_reduce.go deleted file mode 100755 index d883e38..0000000 --- a/external/yq/pkg/yqlib/operator_reduce.go +++ /dev/null @@ -1,57 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" -) - -func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- reduceOp") - //.a as $var reduce (0; . + $var) - //lhs is the assignment operator - //rhs is the reduce block - // '.' refers to the current accumulator, initialised to 0 - // $var references a single element from the .a - - //ensure lhs is actually an assignment - //and rhs is a block (empty) - if expressionNode.LHS.Operation.OperationType != assignVariableOpType { - return Context{}, fmt.Errorf("reduce must be given a variables assignment, got %v instead", expressionNode.LHS.Operation.OperationType.Type) - } else if expressionNode.RHS.Operation.OperationType != blockOpType { - return Context{}, fmt.Errorf("reduce must be given a block, got %v instead", expressionNode.RHS.Operation.OperationType.Type) - } - - arrayExpNode := expressionNode.LHS.LHS - array, err := d.GetMatchingNodes(context, arrayExpNode) - - if err != nil { - return Context{}, err - } - - variableName := expressionNode.LHS.RHS.Operation.StringValue - - initExp := expressionNode.RHS.LHS - - accum, err := d.GetMatchingNodes(context, initExp) - if err != nil { - return Context{}, err - } - - log.Debugf("with variable %v", variableName) - - blockExp := expressionNode.RHS.RHS - for el := array.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debugf("REDUCING WITH %v", NodeToString(candidate)) - l := list.New() - l.PushBack(candidate) - accum.SetVariable(variableName, l) - - accum, err = d.GetMatchingNodes(accum, blockExp) - if err != nil { - return Context{}, err - } - } - - return accum, nil -} diff --git a/external/yq/pkg/yqlib/operator_reduce_test.go b/external/yq/pkg/yqlib/operator_reduce_test.go deleted file mode 100755 index 80b45a5..0000000 --- a/external/yq/pkg/yqlib/operator_reduce_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var reduceOperatorScenarios = []expressionScenario{ - { - description: "Sum numbers", - document: `[10,2, 5, 3]`, - expression: `.[] as $item ireduce (0; . + $item)`, - expected: []string{ - "D0, P[], (!!int)::20\n", - }, - }, - { - description: "Merge all yaml files together", - document: `a: cat`, - document2: `b: dog`, - expression: `. as $item ireduce ({}; . * $item )`, - expected: []string{ - "D0, P[], (!!map)::a: cat\nb: dog\n", - }, - }, - { - description: "Convert an array to an object", - document: `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`, - expression: `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`, - expected: []string{ - "D0, P[], (!!map)::Cathy: apples\nBob: bananas\n", - }, - }, -} - -func TestReduceOperatorScenarios(t *testing.T) { - for _, tt := range reduceOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "reduce", reduceOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_reverse.go b/external/yq/pkg/yqlib/operator_reverse.go deleted file mode 100755 index 1693308..0000000 --- a/external/yq/pkg/yqlib/operator_reverse.go +++ /dev/null @@ -1,34 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - results := list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - candidateNode := unwrapDoc(candidate.Node) - - if candidateNode.Kind != yaml.SequenceNode { - return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag()) - } - - reverseList := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Style: candidateNode.Style} - reverseList.Content = make([]*yaml.Node, len(candidateNode.Content)) - - for i, originalNode := range candidateNode.Content { - reverseList.Content[len(candidateNode.Content)-i-1] = originalNode - } - results.PushBack(candidate.CreateReplacementWithDocWrappers(reverseList)) - - } - - return context.ChildContext(results), nil - -} diff --git a/external/yq/pkg/yqlib/operator_reverse_test.go b/external/yq/pkg/yqlib/operator_reverse_test.go deleted file mode 100755 index 3d67742..0000000 --- a/external/yq/pkg/yqlib/operator_reverse_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package yqlib - -import "testing" - -var reverseOperatorScenarios = []expressionScenario{ - { - description: "Reverse", - document: "[1, 2, 3]", - expression: `reverse`, - expected: []string{ - "D0, P[], (!!seq)::[3, 2, 1]\n", - }, - }, - { - skipDoc: true, - document: "[]", - expression: `reverse`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - skipDoc: true, - document: "[1]", - expression: `reverse`, - expected: []string{ - "D0, P[], (!!seq)::[1]\n", - }, - }, - { - skipDoc: true, - document: "[1,2]", - expression: `reverse`, - expected: []string{ - "D0, P[], (!!seq)::[2, 1]\n", - }, - }, - { - description: "Sort descending by string field", - subdescription: "Use sort with reverse to sort in descending order.", - document: "[{a: banana},{a: cat},{a: apple}]", - expression: `sort_by(.a) | reverse`, - expected: []string{ - "D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n", - }, - }, - { - description: "Sort descending by string field, with comments", - skipDoc: true, - document: "# abc\n[{a: banana},{a: cat},{a: apple}]\n# xyz", - expression: `sort_by(.a) | reverse`, - expected: []string{ - "D0, P[], (!!seq)::# abc\n[{a: cat}, {a: banana}, {a: apple}]\n# xyz\n", - }, - }, -} - -func TestReverseOperatorScenarios(t *testing.T) { - for _, tt := range reverseOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "reverse", reverseOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_select.go b/external/yq/pkg/yqlib/operator_select.go deleted file mode 100755 index 57b9e2c..0000000 --- a/external/yq/pkg/yqlib/operator_select.go +++ /dev/null @@ -1,41 +0,0 @@ -package yqlib - -import ( - "container/list" -) - -func selectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- selectOperation") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - - // find any truthy node - var errDecoding error - includeResult := false - - for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() { - result := resultEl.Value.(*CandidateNode) - includeResult, errDecoding = isTruthy(result) - log.Debugf("isTruthy %v", includeResult) - if errDecoding != nil { - return Context{}, errDecoding - } - if includeResult { - break - } - } - - if includeResult { - results.PushBack(candidate) - } - } - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_select_test.go b/external/yq/pkg/yqlib/operator_select_test.go deleted file mode 100755 index 24702ab..0000000 --- a/external/yq/pkg/yqlib/operator_select_test.go +++ /dev/null @@ -1,150 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var selectOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `cat`, - expression: `select(false, true)`, - expected: []string{ - "D0, P[], (doc)::cat\n", - }, - }, - { - skipDoc: true, - document: `cat`, - expression: `select(true, false)`, - expected: []string{ - "D0, P[], (doc)::cat\n", - }, - }, - { - skipDoc: true, - document: `cat`, - expression: `select(false)`, - expected: []string{}, - }, - { - description: "Select elements from array using wildcard prefix", - document: `[cat,goat,dog]`, - expression: `.[] | select(. == "*at")`, - expected: []string{ - "D0, P[0], (!!str)::cat\n", - "D0, P[1], (!!str)::goat\n", - }, - }, - { - description: "Select elements from array using wildcard suffix", - document: `[go-kart,goat,dog]`, - expression: `.[] | select(. == "go*")`, - expected: []string{ - "D0, P[0], (!!str)::go-kart\n", - "D0, P[1], (!!str)::goat\n", - }, - }, - { - description: "Select elements from array using wildcard prefix and suffix", - document: `[ago, go, meow, going]`, - expression: `.[] | select(. == "*go*")`, - expected: []string{ - "D0, P[0], (!!str)::ago\n", - "D0, P[1], (!!str)::go\n", - "D0, P[3], (!!str)::going\n", - }, - }, - { - description: "Select elements from array with regular expression", - subdescription: "See more regular expression examples under the `string` operator docs.", - document: `[this_0, not_this, nor_0_this, thisTo_4]`, - expression: `.[] | select(test("[a-zA-Z]+_[0-9]$"))`, - expected: []string{ - "D0, P[0], (!!str)::this_0\n", - "D0, P[3], (!!str)::thisTo_4\n", - }, - }, - { - skipDoc: true, - document: "a: hello", - document2: "b: world", - expression: `select(.a == "hello" or .b == "world")`, - expected: []string{ - "D0, P[], (doc)::a: hello\n", - "D0, P[], (doc)::b: world\n", - }, - }, - { - skipDoc: true, - document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`, - expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`, - expected: []string{ - "D0, P[], (doc)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n", - }, - }, - { - skipDoc: true, - document: `[hot, fot, dog]`, - expression: `.[] | select(. == "*at")`, - expected: []string{}, - }, - { - skipDoc: true, - document: `a: [cat,goat,dog]`, - expression: `.a.[] | select(. == "*at")`, - expected: []string{ - "D0, P[a 0], (!!str)::cat\n", - "D0, P[a 1], (!!str)::goat\n"}, - }, - { - description: "Select items from a map", - document: `{ things: cat, bob: goat, horse: dog }`, - expression: `.[] | select(. == "cat" or test("og$"))`, - expected: []string{ - "D0, P[things], (!!str)::cat\n", - "D0, P[horse], (!!str)::dog\n", - }, - }, - { - description: "Use select and with_entries to filter map keys", - document: `{name: bob, legs: 2, game: poker}`, - expression: `with_entries(select(.key | test("ame$")))`, - expected: []string{ - "D0, P[], (!!map)::name: bob\ngame: poker\n", - }, - }, - { - description: "Select multiple items in a map and update", - subdescription: "Note the brackets around the entire LHS.", - document: `a: { things: cat, bob: goat, horse: dog }`, - expression: `(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"`, - expected: []string{ - "D0, P[], (doc)::a: {things: rabbit, bob: rabbit, horse: dog}\n", - }, - }, - { - skipDoc: true, - document: `a: { things: {include: true}, notMe: {include: false}, andMe: {include: fold} }`, - expression: `.a.[] | select(.include)`, - expected: []string{ - "D0, P[a things], (!!map)::{include: true}\n", - "D0, P[a andMe], (!!map)::{include: fold}\n", - }, - }, - { - skipDoc: true, - document: `[cat,~,dog]`, - expression: `.[] | select(. == ~)`, - expected: []string{ - "D0, P[1], (!!null)::~\n", - }, - }, -} - -func TestSelectOperatorScenarios(t *testing.T) { - for _, tt := range selectOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "select", selectOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_self.go b/external/yq/pkg/yqlib/operator_self.go deleted file mode 100755 index 232f7d7..0000000 --- a/external/yq/pkg/yqlib/operator_self.go +++ /dev/null @@ -1,5 +0,0 @@ -package yqlib - -func selfOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - return context, nil -} diff --git a/external/yq/pkg/yqlib/operator_slice.go b/external/yq/pkg/yqlib/operator_slice.go deleted file mode 100755 index e20311d..0000000 --- a/external/yq/pkg/yqlib/operator_slice.go +++ /dev/null @@ -1,72 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, expressionNode *ExpressionNode) (int, error) { - result, err := d.GetMatchingNodes(context.SingleChildContext(node), expressionNode) - if err != nil { - return 0, err - } - if result.MatchingNodes.Len() != 1 { - return 0, fmt.Errorf("expected to find 1 number, got %v instead", result.MatchingNodes.Len()) - } - return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value) -} - -func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debug("slice array operator!") - log.Debug("lhs: %v", expressionNode.LHS.Operation.toString()) - log.Debug("rhs: %v", expressionNode.RHS.Operation.toString()) - - results := list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - lhsNode := el.Value.(*CandidateNode) - original := unwrapDoc(lhsNode.Node) - - firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - relativeFirstNumber := firstNumber - if relativeFirstNumber < 0 { - relativeFirstNumber = len(original.Content) + firstNumber - } - - secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS) - if err != nil { - return Context{}, err - } - - relativeSecondNumber := secondNumber - if relativeSecondNumber < 0 { - relativeSecondNumber = len(original.Content) + secondNumber - } - - log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber) - - var newResults []*yaml.Node - for i := relativeFirstNumber; i < relativeSecondNumber; i++ { - newResults = append(newResults, original.Content[i]) - } - - slicedArrayNode := &yaml.Node{ - Kind: yaml.SequenceNode, - Tag: original.Tag, - Content: newResults, - } - results.PushBack(lhsNode.CreateReplacement(slicedArrayNode)) - - } - - // result is now the context that has the nodes we need to put back into a sequence. - //what about multiple arrays in the context? I think we need to create an array for each one - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_slice_test.go b/external/yq/pkg/yqlib/operator_slice_test.go deleted file mode 100755 index 9785410..0000000 --- a/external/yq/pkg/yqlib/operator_slice_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package yqlib - -import "testing" - -var sliceArrayScenarios = []expressionScenario{ - { - description: "Slicing arrays", - document: `[cat, dog, frog, cow]`, - expression: `.[1:3]`, - expected: []string{ - "D0, P[], (!!seq)::- dog\n- frog\n", - }, - }, - { - description: "Slicing arrays - without the first number", - subdescription: "Starts from the start of the array", - document: `[cat, dog, frog, cow]`, - expression: `.[:2]`, - expected: []string{ - "D0, P[], (!!seq)::- cat\n- dog\n", - }, - }, - { - description: "Slicing arrays - without the second number", - subdescription: "Finishes at the end of the array", - document: `[cat, dog, frog, cow]`, - expression: `.[2:]`, - expected: []string{ - "D0, P[], (!!seq)::- frog\n- cow\n", - }, - }, - { - description: "Slicing arrays - use negative numbers to count backwards from the end", - document: `[cat, dog, frog, cow]`, - expression: `.[1:-1]`, - expected: []string{ - "D0, P[], (!!seq)::- dog\n- frog\n", - }, - }, - { - description: "Inserting into the middle of an array", - subdescription: "using an expression to find the index", - document: `[cat, dog, frog, cow]`, - expression: `(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]`, - expected: []string{ - "D0, P[], (!!seq)::- cat\n- dog\n- rabbit\n- frog\n- cow\n", - }, - }, - { - skipDoc: true, - document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`, - expression: `.[] | .[1:3]`, - expected: []string{ - "D0, P[0], (!!seq)::- dog\n- frog\n", - "D0, P[1], (!!seq)::- banana\n- grape\n", - }, - }, - { - skipDoc: true, - document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`, - expression: `.[] | .[-2:-1]`, - expected: []string{ - "D0, P[0], (!!seq)::- frog\n", - "D0, P[1], (!!seq)::- grape\n", - }, - }, - { - skipDoc: true, - document: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`, - expression: `.[10:11]`, - expected: []string{ - "D0, P[], (!!seq)::- cat11\n", - }, - }, - { - skipDoc: true, - document: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`, - expression: `.[-11:-10]`, - expected: []string{ - "D0, P[], (!!seq)::- cat1\n", - }, - }, -} - -func TestSliceOperatorScenarios(t *testing.T) { - for _, tt := range sliceArrayScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "slice-array", sliceArrayScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_sort.go b/external/yq/pkg/yqlib/operator_sort.go deleted file mode 100755 index 62c85e5..0000000 --- a/external/yq/pkg/yqlib/operator_sort.go +++ /dev/null @@ -1,165 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "sort" - "strconv" - "strings" - "time" - - yaml "gopkg.in/yaml.v3" -) - -func sortOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} - expressionNode.RHS = selfExpression - return sortByOperator(d, context, expressionNode) -} - -// context represents the current matching nodes in the expression pipeline -// expressionNode is your current expression (sort_by) -func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - results := list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - candidateNode := unwrapDoc(candidate.Node) - - if candidateNode.Kind != yaml.SequenceNode { - return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag()) - } - - sortableArray := make(sortableNodeArray, len(candidateNode.Content)) - - for i, originalNode := range candidateNode.Content { - - childCandidate := candidate.CreateChildInArray(i, originalNode) - compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(childCandidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - nodeToCompare := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"} - if compareContext.MatchingNodes.Len() > 0 { - nodeToCompare = compareContext.MatchingNodes.Front().Value.(*CandidateNode).Node - } - - log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare))) - - sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()} - - if nodeToCompare.Kind != yaml.ScalarNode { - return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag) - } - - } - - sort.Stable(sortableArray) - - sortedList := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Style: candidateNode.Style} - sortedList.Content = make([]*yaml.Node, len(candidateNode.Content)) - - for i, sortedNode := range sortableArray { - sortedList.Content[i] = sortedNode.Node - } - results.PushBack(candidate.CreateReplacementWithDocWrappers(sortedList)) - } - return context.ChildContext(results), nil -} - -type sortableNode struct { - Node *yaml.Node - NodeToCompare *yaml.Node - dateTimeLayout string -} - -type sortableNodeArray []sortableNode - -func (a sortableNodeArray) Len() int { return len(a) } -func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func (a sortableNodeArray) Less(i, j int) bool { - lhs := a[i].NodeToCompare - rhs := a[j].NodeToCompare - - lhsTag := lhs.Tag - rhsTag := rhs.Tag - - if !strings.HasPrefix(lhsTag, "!!") { - // custom tag - we have to have a guess - lhsTag = guessTagFromCustomType(lhs) - } - - if !strings.HasPrefix(rhsTag, "!!") { - // custom tag - we have to have a guess - rhsTag = guessTagFromCustomType(rhs) - } - - isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp" - layout := a[i].dateTimeLayout - // if the lhs is a string, it might be a timestamp in a custom format. - if lhsTag == "!!str" && layout != time.RFC3339 { - _, errLhs := parseDateTime(layout, lhs.Value) - _, errRhs := parseDateTime(layout, rhs.Value) - isDateTime = errLhs == nil && errRhs == nil - } - - if lhsTag == "!!null" && rhsTag != "!!null" { - return true - } else if lhsTag != "!!null" && rhsTag == "!!null" { - return false - } else if lhsTag == "!!bool" && rhsTag != "!!bool" { - return true - } else if lhsTag != "!!bool" && rhsTag == "!!bool" { - return false - } else if lhsTag == "!!bool" && rhsTag == "!!bool" { - lhsTruthy, err := isTruthyNode(lhs) - if err != nil { - panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err)) - } - - rhsTruthy, err := isTruthyNode(rhs) - if err != nil { - panic(fmt.Errorf("could not parse %v as boolean: %w", rhs.Value, err)) - } - - return !lhsTruthy && rhsTruthy - } else if isDateTime { - lhsTime, err := parseDateTime(layout, lhs.Value) - if err != nil { - log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err) - return strings.Compare(lhs.Value, rhs.Value) < 0 - } - rhsTime, err := parseDateTime(layout, rhs.Value) - if err != nil { - log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err) - return strings.Compare(lhs.Value, rhs.Value) < 0 - } - return lhsTime.Before(rhsTime) - } else if lhsTag == "!!int" && rhsTag == "!!int" { - _, lhsNum, err := parseInt64(lhs.Value) - if err != nil { - panic(err) - } - _, rhsNum, err := parseInt64(rhs.Value) - if err != nil { - panic(err) - } - return lhsNum < rhsNum - } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { - lhsNum, err := strconv.ParseFloat(lhs.Value, 64) - if err != nil { - panic(err) - } - rhsNum, err := strconv.ParseFloat(rhs.Value, 64) - if err != nil { - panic(err) - } - return lhsNum < rhsNum - } - - return strings.Compare(lhs.Value, rhs.Value) < 0 -} diff --git a/external/yq/pkg/yqlib/operator_sort_keys.go b/external/yq/pkg/yqlib/operator_sort_keys.go deleted file mode 100755 index 0ab1a3c..0000000 --- a/external/yq/pkg/yqlib/operator_sort_keys.go +++ /dev/null @@ -1,52 +0,0 @@ -package yqlib - -import ( - "sort" - - yaml "gopkg.in/yaml.v3" -) - -func sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - for childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() { - node := unwrapDoc(childEl.Value.(*CandidateNode).Node) - if node.Kind == yaml.MappingNode { - sortKeys(node) - } - if err != nil { - return Context{}, err - } - } - - } - return context, nil -} - -func sortKeys(node *yaml.Node) { - keys := make([]string, len(node.Content)/2) - keyBucket := map[string]*yaml.Node{} - valueBucket := map[string]*yaml.Node{} - var contents = node.Content - for index := 0; index < len(contents); index = index + 2 { - key := contents[index] - value := contents[index+1] - keys[index/2] = key.Value - keyBucket[key.Value] = key - valueBucket[key.Value] = value - } - sort.Strings(keys) - sortedContent := make([]*yaml.Node, len(node.Content)) - for index := 0; index < len(keys); index = index + 1 { - keyString := keys[index] - sortedContent[index*2] = keyBucket[keyString] - sortedContent[1+(index*2)] = valueBucket[keyString] - } - node.Content = sortedContent -} diff --git a/external/yq/pkg/yqlib/operator_sort_keys_test.go b/external/yq/pkg/yqlib/operator_sort_keys_test.go deleted file mode 100755 index 8d42155..0000000 --- a/external/yq/pkg/yqlib/operator_sort_keys_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var sortKeysOperatorScenarios = []expressionScenario{ - { - description: "Sort keys of map", - document: `{c: frog, a: blah, b: bing}`, - expression: `sort_keys(.)`, - expected: []string{ - "D0, P[], (doc)::{a: blah, b: bing, c: frog}\n", - }, - }, - { - skipDoc: true, - document: `{c: frog}`, - expression: `sort_keys(.d)`, - expected: []string{ - "D0, P[], (doc)::{c: frog}\n", - }, - }, - { - description: "Sort keys recursively", - subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted", - document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`, - expression: `sort_keys(..)`, - expected: []string{ - "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", - }, - }, -} - -func TestSortKeysOperatorScenarios(t *testing.T) { - for _, tt := range sortKeysOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "sort-keys", sortKeysOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_sort_test.go b/external/yq/pkg/yqlib/operator_sort_test.go deleted file mode 100755 index 21e5c99..0000000 --- a/external/yq/pkg/yqlib/operator_sort_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package yqlib - -import "testing" - -var sortByOperatorScenarios = []expressionScenario{ - { - description: "Sort by string field", - document: "[{a: banana},{a: cat},{a: apple}]", - expression: `sort_by(.a)`, - expected: []string{ - "D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n", - }, - }, - { - description: "Sort descending by string field", - subdescription: "Use sort with reverse to sort in descending order.", - document: "[{a: banana},{a: cat},{a: apple}]", - expression: `sort_by(.a) | reverse`, - expected: []string{ - "D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n", - }, - }, - { - description: "Sort array in place", - document: "cool: [{a: banana},{a: cat},{a: apple}]", - expression: `.cool |= sort_by(.a)`, - expected: []string{ - "D0, P[], (doc)::cool: [{a: apple}, {a: banana}, {a: cat}]\n", - }, - }, - { - description: "Sort array of objects by key", - subdescription: "Note that you can give sort_by complex expressions, not just paths", - document: "cool: [{b: banana},{a: banana},{c: banana}]", - expression: `.cool |= sort_by(keys | .[0])`, - expected: []string{ - "D0, P[], (doc)::cool: [{a: banana}, {b: banana}, {c: banana}]\n", - }, - }, - { - description: "Sort is stable", - subdescription: "Note the order of the elements in unchanged when equal in sorting.", - document: "[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]", - expression: `sort_by(.a)`, - expected: []string{ - "D0, P[], (!!seq)::[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\n", - }, - }, - { - description: "Sort by numeric field", - document: "[{a: 10},{a: 100},{a: 1}]", - expression: `sort_by(.a)`, - expected: []string{ - "D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n", - }, - }, - { - description: "Sort by custom date field", - document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`, - expression: `with_dtf("02-Jan-2006"; sort_by(.a))`, - expected: []string{ - "D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n", - }, - }, - { - skipDoc: true, - document: "[{a: 1.1},{a: 1.001},{a: 1.01}]", - expression: `sort_by(.a)`, - expected: []string{ - "D0, P[], (!!seq)::[{a: 1.001}, {a: 1.01}, {a: 1.1}]\n", - }, - }, - { - description: "Sort, nulls come first", - document: "[8,3,null,6, true, false, cat]", - expression: `sort`, - expected: []string{ - "D0, P[], (!!seq)::[null, false, true, 3, 6, 8, cat]\n", - }, - }, - { - skipDoc: true, - description: "false before true", - document: "[{a: false, b: 1}, {a: true, b: 2}, {a: false, b: 3}]", - expression: `sort_by(.a)`, - expected: []string{ - "D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\n", - }, - }, - { - skipDoc: true, - description: "head comment", - document: "# abc\n- def\n# ghi", - expression: `sort`, - expected: []string{ - "D0, P[], (!!seq)::# abc\n- def\n# ghi\n", - }, - }, -} - -func TestSortByOperatorScenarios(t *testing.T) { - for _, tt := range sortByOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "sort", sortByOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_split_document.go b/external/yq/pkg/yqlib/operator_split_document.go deleted file mode 100755 index eba2496..0000000 --- a/external/yq/pkg/yqlib/operator_split_document.go +++ /dev/null @@ -1,14 +0,0 @@ -package yqlib - -func splitDocumentOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- splitDocumentOperator") - - var index uint - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidate.Document = index - index = index + 1 - } - - return context, nil -} diff --git a/external/yq/pkg/yqlib/operator_split_document_test.go b/external/yq/pkg/yqlib/operator_split_document_test.go deleted file mode 100755 index a2c4f90..0000000 --- a/external/yq/pkg/yqlib/operator_split_document_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var splitDocOperatorScenarios = []expressionScenario{ - { - description: "Split empty", - document: ``, - expression: `split_doc`, - expected: []string{ - "D0, P[], (!!null)::\n", - }, - }, - { - description: "Split array", - document: `[{a: cat}, {b: dog}]`, - expression: `.[] | split_doc`, - expected: []string{ - "D0, P[0], (!!map)::{a: cat}\n", - "D1, P[1], (!!map)::{b: dog}\n", - }, - }, -} - -func TestSplitDocOperatorScenarios(t *testing.T) { - for _, tt := range splitDocOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "split-into-documents", splitDocOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_strings.go b/external/yq/pkg/yqlib/operator_strings.go deleted file mode 100755 index 0ccfbfe..0000000 --- a/external/yq/pkg/yqlib/operator_strings.go +++ /dev/null @@ -1,444 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - "regexp" - "strings" - - "gopkg.in/yaml.v3" -) - -type changeCasePrefs struct { - ToUpperCase bool -} - -func trimSpaceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - results := list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot trim %v, can only operate on strings. ", node.Tag) - } - - newStringNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.Tag, Style: node.Style} - newStringNode.Value = strings.TrimSpace(node.Value) - results.PushBack(candidate.CreateReplacement(newStringNode)) - - } - - return context.ChildContext(results), nil -} - -func changeCaseOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - results := list.New() - prefs := expressionNode.Operation.Preferences.(changeCasePrefs) - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot change case with %v, can only operate on strings. ", node.Tag) - } - - newStringNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: node.Tag, Style: node.Style} - if prefs.ToUpperCase { - newStringNode.Value = strings.ToUpper(node.Value) - } else { - newStringNode.Value = strings.ToLower(node.Value) - } - results.PushBack(candidate.CreateReplacement(newStringNode)) - - } - - return context.ChildContext(results), nil - -} - -func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) { - regEx := "" - replacementText := "" - - regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) - if err != nil { - return "", "", err - } - if regExNodes.MatchingNodes.Front() != nil { - regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - - log.Debug("regEx %v", regEx) - - replacementNodes, err := d.GetMatchingNodes(context, block.RHS) - if err != nil { - return "", "", err - } - if replacementNodes.MatchingNodes.Front() != nil { - replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - - return regEx, replacementText, nil -} - -func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node { - replacedString := regex.ReplaceAllString(original, replacement) - return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"} -} - -func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - //rhs block operator - //lhs of block = regex - //rhs of block = replacement expression - block := expressionNode.RHS - - regExStr, replacementText, err := getSubstituteParameters(d, block, context) - - if err != nil { - return Context{}, err - } - - regEx, err := regexp.Compile(regExStr) - if err != nil { - return Context{}, err - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) - } - - targetNode := substitute(node.Value, regEx, replacementText) - result := candidate.CreateReplacement(targetNode) - results.PushBack(result) - } - - return context.ChildContext(results), nil - -} - -func addMatch(original []*yaml.Node, match string, offset int, name string) []*yaml.Node { - - newContent := append(original, - createScalarNode("string", "string")) - - if offset < 0 { - // offset of -1 means there was no match, force a null value like jq - newContent = append(newContent, - createScalarNode(nil, "null"), - ) - } else { - newContent = append(newContent, - createScalarNode(match, match), - ) - } - - newContent = append(newContent, - createScalarNode("offset", "offset"), - createScalarNode(offset, fmt.Sprintf("%v", offset)), - createScalarNode("length", "length"), - createScalarNode(len(match), fmt.Sprintf("%v", len(match)))) - - if name != "" { - newContent = append(newContent, - createScalarNode("name", "name"), - createScalarNode(name, name), - ) - } - return newContent -} - -type matchPreferences struct { - Global bool -} - -func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) { - var allMatches [][]string - var allIndices [][]int - - if matchPrefs.Global { - allMatches = regEx.FindAllStringSubmatch(value, -1) - allIndices = regEx.FindAllStringSubmatchIndex(value, -1) - } else { - allMatches = [][]string{regEx.FindStringSubmatch(value)} - allIndices = [][]int{regEx.FindStringSubmatchIndex(value)} - } - - log.Debug("allMatches, %v", allMatches) - return allMatches, allIndices -} - -func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) { - subNames := regEx.SubexpNames() - allMatches, allIndices := getMatches(matchPrefs, regEx, value) - - // if all matches just has an empty array in it, - // then nothing matched - if len(allMatches) > 0 && len(allMatches[0]) == 0 { - return - } - - for i, matches := range allMatches { - capturesListNode := &yaml.Node{Kind: yaml.SequenceNode} - match, submatches := matches[0], matches[1:] - for j, submatch := range submatches { - captureNode := &yaml.Node{Kind: yaml.MappingNode} - captureNode.Content = addMatch(captureNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1]) - capturesListNode.Content = append(capturesListNode.Content, captureNode) - } - - node := &yaml.Node{Kind: yaml.MappingNode} - node.Content = addMatch(node.Content, match, allIndices[i][0], "") - node.Content = append(node.Content, - createScalarNode("captures", "captures"), - capturesListNode, - ) - results.PushBack(candidate.CreateReplacement(node)) - - } - -} - -func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) { - subNames := regEx.SubexpNames() - allMatches, allIndices := getMatches(matchPrefs, regEx, value) - - // if all matches just has an empty array in it, - // then nothing matched - if len(allMatches) > 0 && len(allMatches[0]) == 0 { - return - } - - for i, matches := range allMatches { - capturesNode := &yaml.Node{Kind: yaml.MappingNode} - - _, submatches := matches[0], matches[1:] - for j, submatch := range submatches { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(subNames[j+1], subNames[j+1])) - - offset := allIndices[i][2+j*2] - // offset of -1 means there was no match, force a null value like jq - if offset < 0 { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(nil, "null"), - ) - } else { - capturesNode.Content = append(capturesNode.Content, - createScalarNode(submatch, submatch), - ) - } - } - - results.PushBack(candidate.CreateReplacement(capturesNode)) - - } - -} - -func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) { - regExExpNode := expressionNode.RHS - - matchPrefs := matchPreferences{} - - // we got given parameters e.g. match(exp; params) - if expressionNode.RHS.Operation.OperationType == blockOpType { - block := expressionNode.RHS - regExExpNode = block.LHS - replacementNodes, err := d.GetMatchingNodes(context, block.RHS) - if err != nil { - return nil, matchPrefs, err - } - paramText := "" - if replacementNodes.MatchingNodes.Front() != nil { - paramText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - if strings.Contains(paramText, "g") { - paramText = strings.ReplaceAll(paramText, "g", "") - matchPrefs.Global = true - } - if strings.Contains(paramText, "i") { - return nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match("(?i)cat")`) - } - if len(paramText) > 0 { - return nil, matchPrefs, fmt.Errorf(`Unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText) - } - } - - regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode) - if err != nil { - return nil, matchPrefs, err - } - log.Debug(NodesToString(regExNodes.MatchingNodes)) - regExStr := "" - if regExNodes.MatchingNodes.Front() != nil { - regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - log.Debug("regEx %v", regExStr) - regEx, err := regexp.Compile(regExStr) - return regEx, matchPrefs, err -} - -func matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode) - if err != nil { - return Context{}, err - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) - } - - match(matchPrefs, regEx, candidate, node.Value, results) - } - - return context.ChildContext(results), nil -} - -func captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode) - if err != nil { - return Context{}, err - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) - } - capture(matchPrefs, regEx, candidate, node.Value, results) - - } - - return context.ChildContext(results), nil -} - -func testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - regEx, _, err := extractMatchArguments(d, context, expressionNode) - if err != nil { - return Context{}, err - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) - } - matches := regEx.FindStringSubmatch(node.Value) - results.PushBack(createBooleanCandidate(candidate, len(matches) > 0)) - - } - - return context.ChildContext(results), nil -} - -func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- joinStringOperator") - joinStr := "" - - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if rhs.MatchingNodes.Front() != nil { - joinStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - if node.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag) - } - targetNode := join(node.Content, joinStr) - result := candidate.CreateReplacement(targetNode) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} - -func join(content []*yaml.Node, joinStr string) *yaml.Node { - var stringsToJoin []string - for _, node := range content { - str := node.Value - if node.Tag == "!!null" { - str = "" - } - stringsToJoin = append(stringsToJoin, str) - } - - return &yaml.Node{Kind: yaml.ScalarNode, Value: strings.Join(stringsToJoin, joinStr), Tag: "!!str"} -} - -func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- splitStringOperator") - splitStr := "" - - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if rhs.MatchingNodes.Front() != nil { - splitStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := unwrapDoc(candidate.Node) - if node.Tag == "!!null" { - continue - } - - if guessTagFromCustomType(node) != "!!str" { - return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag) - } - targetNode := split(node.Value, splitStr) - result := candidate.CreateReplacement(targetNode) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} - -func split(value string, spltStr string) *yaml.Node { - var contents []*yaml.Node - - if value != "" { - var newStrings = strings.Split(value, spltStr) - contents = make([]*yaml.Node, len(newStrings)) - - for index, str := range newStrings { - contents[index] = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: str} - } - } - - return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents} -} diff --git a/external/yq/pkg/yqlib/operator_strings_test.go b/external/yq/pkg/yqlib/operator_strings_test.go deleted file mode 100755 index ee97ede..0000000 --- a/external/yq/pkg/yqlib/operator_strings_test.go +++ /dev/null @@ -1,277 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var stringsOperatorScenarios = []expressionScenario{ - { - description: "To up (upper) case", - subdescription: "Works with unicode characters", - document: `água`, - expression: "upcase", - expected: []string{ - "D0, P[], (!!str)::ÁGUA\n", - }, - }, - { - skipDoc: true, - document: `!camel água`, - expression: "upcase", - expected: []string{ - "D0, P[], (!camel)::ÁGUA\n", - }, - }, - { - description: "To down (lower) case", - subdescription: "Works with unicode characters", - document: `ÁgUA`, - expression: "downcase", - expected: []string{ - "D0, P[], (!!str)::água\n", - }, - }, - { - skipDoc: true, - document: `!camel ÁgUA`, - expression: "downcase", - expected: []string{ - "D0, P[], (!camel)::água\n", - }, - }, - { - description: "Join strings", - document: `[cat, meow, 1, null, true]`, - expression: `join("; ")`, - expected: []string{ - "D0, P[], (!!str)::cat; meow; 1; ; true\n", - }, - }, - { - description: "Trim strings", - document: `[" cat", "dog ", " cow cow ", horse]`, - expression: `.[] | trim`, - expected: []string{ - "D0, P[0], (!!str)::cat\n", - "D0, P[1], (!!str)::dog\n", - "D0, P[2], (!!str)::cow cow\n", - "D0, P[3], (!!str)::horse\n", - }, - }, - { - skipDoc: true, - document: `[!horse cat, !goat meow, !frog 1, null, true]`, - expression: `join("; ")`, - expected: []string{ - "D0, P[], (!!str)::cat; meow; 1; ; true\n", - }, - }, - { - description: "Match string", - document: `foo bar foo`, - expression: `match("foo")`, - expected: []string{ - "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", - }, - }, - { - skipDoc: true, - document: `!horse foo bar foo`, - expression: `match("foo")`, - expected: []string{ - "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", - }, - }, - { - description: "Match string, case insensitive", - document: `foo bar FOO`, - expression: `[match("(?i)foo"; "g")]`, - expected: []string{ - "D0, P[], (!!seq)::- string: foo\n offset: 0\n length: 3\n captures: []\n- string: FOO\n offset: 8\n length: 3\n captures: []\n", - }, - }, - { - description: "Match with global capture group", - document: `abc abc`, - expression: `[match("(ab)(c)"; "g")]`, - expected: []string{ - "D0, P[], (!!seq)::- string: abc\n offset: 0\n length: 3\n captures:\n - string: ab\n offset: 0\n length: 2\n - string: c\n offset: 2\n length: 1\n- string: abc\n offset: 4\n length: 3\n captures:\n - string: ab\n offset: 4\n length: 2\n - string: c\n offset: 6\n length: 1\n", - }, - }, - { - description: "Match with named capture groups", - document: `foo bar foo foo foo`, - expression: `[match("foo (?Pbar)? foo"; "g")]`, - expected: []string{ - "D0, P[], (!!seq)::- string: foo bar foo\n offset: 0\n length: 11\n captures:\n - string: bar\n offset: 4\n length: 3\n name: bar123\n- string: foo foo\n offset: 12\n length: 8\n captures:\n - string: null\n offset: -1\n length: 0\n name: bar123\n", - }, - }, - { - description: "Capture named groups into a map", - document: `xyzzy-14`, - expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, - expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\n", - }, - }, - { - skipDoc: true, - document: `!horse xyzzy-14`, - expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, - expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\n", - }, - }, - { - skipDoc: true, - description: "Capture named groups into a map, with null", - document: `xyzzy-14`, - expression: `capture("(?P[a-z]+)-(?P[0-9]+)(?Pbar)?")`, - expected: []string{ - "D0, P[], ()::a: xyzzy\nn: \"14\"\nbar123: null\n", - }, - }, - { - description: "Match without global flag", - document: `cat cat`, - expression: `match("cat")`, - expected: []string{ - "D0, P[], ()::string: cat\noffset: 0\nlength: 3\ncaptures: []\n", - }, - }, - { - description: "Match with global flag", - document: `cat cat`, - expression: `[match("cat"; "g")]`, - expected: []string{ - "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", - }, - }, - { - skipDoc: true, - document: `!horse cat cat`, - expression: `[match("cat"; "g")]`, - expected: []string{ - "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", - }, - }, - { - skipDoc: true, - description: "No match", - document: `dog`, - expression: `match("cat"; "g")`, - expected: []string{}, - }, - { - skipDoc: true, - description: "No match", - expression: `"dog" | match("cat", "g")`, - expected: []string{}, - }, - { - skipDoc: true, - description: "No match", - expression: `"dog" | match("cat")`, - expected: []string{}, - }, - { - description: "Test using regex", - subdescription: "Like jq's equivalent, this works like match but only returns true/false instead of full match details", - document: `["cat", "dog"]`, - expression: `.[] | test("at")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `[!horse "cat", !cat "dog"]`, - expression: `.[] | test("at")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::false\n", - }, - }, - { - skipDoc: true, - document: `["cat*", "cat*", "cat"]`, - expression: `.[] | test("cat\*")`, - expected: []string{ - "D0, P[0], (!!bool)::true\n", - "D0, P[1], (!!bool)::true\n", - "D0, P[2], (!!bool)::false\n", - }, - }, - { - description: "Substitute / Replace string", - subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.", - document: `a: dogs are great`, - expression: `.a |= sub("dogs", "cats")`, - expected: []string{ - "D0, P[], (doc)::a: cats are great\n", - }, - }, - { - description: "Substitute / Replace string with regex", - subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.", - document: "a: cat\nb: heat", - expression: `.[] |= sub("(a)", "${1}r")`, - expected: []string{ - "D0, P[], (doc)::a: cart\nb: heart\n", - }, - }, - { - description: "Custom types: that are really strings", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse cat\nb: !goat heat", - expression: `.[] |= sub("(a)", "${1}r")`, - expected: []string{ - "D0, P[], (doc)::a: !horse cart\nb: !goat heart\n", - }, - }, - { - description: "Split strings", - document: `"cat; meow; 1; ; true"`, - expression: `split("; ")`, - expected: []string{ - "D0, P[], (!!seq)::- cat\n- meow\n- \"1\"\n- \"\"\n- \"true\"\n", - }, - }, - { - description: "Split strings one match", - document: `"word"`, - expression: `split("; ")`, - expected: []string{ - "D0, P[], (!!seq)::- word\n", - }, - }, - { - skipDoc: true, - document: `!horse "word"`, - expression: `split("; ")`, - expected: []string{ - "D0, P[], (!!seq)::- word\n", - }, - }, - { - skipDoc: true, - document: `""`, - expression: `split("; ")`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", // dont actually want this, just not to error - }, - }, - { - skipDoc: true, - expression: `split("; ")`, - expected: []string{}, - }, -} - -func TestStringsOperatorScenarios(t *testing.T) { - for _, tt := range stringsOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "string-operators", stringsOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_style.go b/external/yq/pkg/yqlib/operator_style.go deleted file mode 100755 index ed7f654..0000000 --- a/external/yq/pkg/yqlib/operator_style.go +++ /dev/null @@ -1,108 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - yaml "gopkg.in/yaml.v3" -) - -func parseStyle(customStyle string) (yaml.Style, error) { - if customStyle == "tagged" { - return yaml.TaggedStyle, nil - } else if customStyle == "double" { - return yaml.DoubleQuotedStyle, nil - } else if customStyle == "single" { - return yaml.SingleQuotedStyle, nil - } else if customStyle == "literal" { - return yaml.LiteralStyle, nil - } else if customStyle == "folded" { - return yaml.FoldedStyle, nil - } else if customStyle == "flow" { - return yaml.FlowStyle, nil - } else if customStyle != "" { - return 0, fmt.Errorf("Unknown style %v", customStyle) - } - return 0, nil -} - -func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("AssignStyleOperator: %v") - var style yaml.Style - if !expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - style, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value) - if err != nil { - return Context{}, err - } - } - } - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debugf("Setting style of : %v", candidate.GetKey()) - if expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - style, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value) - if err != nil { - return Context{}, err - } - } - } - - candidate.Node.Style = style - } - - return context, nil -} - -func getStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetStyleOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - var style string - switch candidate.Node.Style { - case yaml.TaggedStyle: - style = "tagged" - case yaml.DoubleQuotedStyle: - style = "double" - case yaml.SingleQuotedStyle: - style = "single" - case yaml.LiteralStyle: - style = "literal" - case yaml.FoldedStyle: - style = "folded" - case yaml.FlowStyle: - style = "flow" - case 0: - style = "" - default: - style = "" - } - node := &yaml.Node{Kind: yaml.ScalarNode, Value: style, Tag: "!!str"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_style_test.go b/external/yq/pkg/yqlib/operator_style_test.go deleted file mode 100755 index ce263a7..0000000 --- a/external/yq/pkg/yqlib/operator_style_test.go +++ /dev/null @@ -1,156 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var styleOperatorScenarios = []expressionScenario{ - { - description: "Update and set style of a particular node (simple)", - document: `a: {b: thing, c: something}`, - expression: `.a.b = "new" | .a.b style="double"`, - expected: []string{ - "D0, P[], (doc)::a: {b: \"new\", c: something}\n", - }, - }, - { - description: "Update and set style of a particular node using path variables", - document: `a: {b: thing, c: something}`, - expression: `with(.a.b ; . = "new" | . style="double")`, - expected: []string{ - "D0, P[], (doc)::a: {b: \"new\", c: something}\n", - }, - }, - { - description: "Set tagged style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="tagged"`, - expected: []string{ - "D0, P[], (!!map)::!!map\na: !!str cat\nb: !!int 5\nc: !!float 3.2\ne: !!bool true\n", - }, - }, - { - description: "Set double quote style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="double"`, - expected: []string{ - "D0, P[], (!!map)::a: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\n", - }, - }, - { - description: "Set double quote style on map keys too", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `... style="double"`, - expected: []string{ - "D0, P[], (!!map)::\"a\": \"cat\"\n\"b\": \"5\"\n\"c\": \"3.2\"\n\"e\": \"true\"\n", - }, - }, - { - skipDoc: true, - document: "bing: &foo frog\na:\n c: cat\n <<: [*foo]", - expression: `(... | select(tag=="!!str")) style="single"`, - expected: []string{ - "D0, P[], (!!map)::'bing': &foo 'frog'\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n", - }, - }, - { - description: "Set single quote style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="single"`, - expected: []string{ - "D0, P[], (!!map)::a: 'cat'\nb: '5'\nc: '3.2'\ne: 'true'\n", - }, - }, - { - description: "Set literal quote style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="literal"`, - expected: []string{ - `D0, P[], (!!map)::a: |- - cat -b: |- - 5 -c: |- - 3.2 -e: |- - true -`, - }, - }, - { - description: "Set folded quote style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="folded"`, - expected: []string{ - `D0, P[], (!!map)::a: >- - cat -b: >- - 5 -c: >- - 3.2 -e: >- - true -`, - }, - }, - { - description: "Set flow quote style", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `.. style="flow"`, - expected: []string{ - "D0, P[], (!!map)::{a: cat, b: 5, c: 3.2, e: true}\n", - }, - }, - { - description: "Reset style - or pretty print", - subdescription: "Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this.", - document: `{a: cat, "b": 5, 'c': 3.2, "e": true}`, - expression: `... style=""`, - expected: []string{ - "D0, P[], (!!map)::a: cat\nb: 5\nc: 3.2\ne: true\n", - }, - }, - { - description: "Set style relatively with assign-update", - document: `{a: single, b: double}`, - expression: `.[] style |= .`, - expected: []string{ - "D0, P[], (doc)::{a: 'single', b: \"double\"}\n", - }, - }, - { - skipDoc: true, - document: `{a: cat, b: double}`, - expression: `.a style=.b`, - expected: []string{ - "D0, P[], (doc)::{a: \"cat\", b: double}\n", - }, - }, - { - description: "Read style", - document: `{a: "cat", b: 'thing'}`, - dontFormatInputForDoc: true, - expression: `.. | style`, - expected: []string{ - "D0, P[], (!!str)::flow\n", - "D0, P[a], (!!str)::double\n", - "D0, P[b], (!!str)::single\n", - }, - }, - { - skipDoc: true, - document: `a: cat`, - expression: `.. | style`, - expected: []string{ - "D0, P[], (!!str)::\n", - "D0, P[a], (!!str)::\n", - }, - }, -} - -func TestStyleOperatorScenarios(t *testing.T) { - for _, tt := range styleOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "style", styleOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_subtract.go b/external/yq/pkg/yqlib/operator_subtract.go deleted file mode 100755 index 90867d4..0000000 --- a/external/yq/pkg/yqlib/operator_subtract.go +++ /dev/null @@ -1,162 +0,0 @@ -package yqlib - -import ( - "fmt" - "strconv" - "strings" - "time" - - "gopkg.in/yaml.v3" -) - -func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { - return &ExpressionNode{Operation: &Operation{OperationType: subtractOpType}, - LHS: lhs, - RHS: rhs} -} - -func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - return compoundAssignFunction(d, context, expressionNode, createSubtractOp) -} - -func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("Subtract operator") - - return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false) -} - -func subtractArray(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - newLHSArray := make([]*yaml.Node, 0) - - for lindex := 0; lindex < len(lhs.Node.Content); lindex = lindex + 1 { - shouldInclude := true - for rindex := 0; rindex < len(rhs.Node.Content) && shouldInclude; rindex = rindex + 1 { - if recursiveNodeEqual(lhs.Node.Content[lindex], rhs.Node.Content[rindex]) { - shouldInclude = false - } - } - if shouldInclude { - newLHSArray = append(newLHSArray, lhs.Node.Content[lindex]) - } - } - lhs.Node.Content = newLHSArray - return lhs, nil -} - -func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - lhs.Node = unwrapDoc(lhs.Node) - rhs.Node = unwrapDoc(rhs.Node) - - lhsNode := lhs.Node - - if lhsNode.Tag == "!!null" { - return lhs.CreateReplacement(rhs.Node), nil - } - - target := lhs.CreateReplacement(&yaml.Node{}) - - switch lhsNode.Kind { - case yaml.MappingNode: - return nil, fmt.Errorf("maps not yet supported for subtraction") - case yaml.SequenceNode: - if rhs.Node.Kind != yaml.SequenceNode { - return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag) - } - return subtractArray(lhs, rhs) - case yaml.ScalarNode: - if rhs.Node.Kind != yaml.ScalarNode { - return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag) - } - target.Node.Kind = yaml.ScalarNode - target.Node.Style = lhsNode.Style - if err := subtractScalars(context, target, lhsNode, rhs.Node); err != nil { - return nil, err - } - } - - return target, nil -} - -func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { - lhsTag := lhs.Tag - rhsTag := rhs.Tag - lhsIsCustom := false - if !strings.HasPrefix(lhsTag, "!!") { - // custom tag - we have to have a guess - lhsTag = guessTagFromCustomType(lhs) - lhsIsCustom = true - } - - if !strings.HasPrefix(rhsTag, "!!") { - // custom tag - we have to have a guess - rhsTag = guessTagFromCustomType(rhs) - } - - isDateTime := lhsTag == "!!timestamp" - // if the lhs is a string, it might be a timestamp in a custom format. - if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { - _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) - isDateTime = err == nil - } - - if isDateTime { - return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs) - } else if lhsTag == "!!str" { - return fmt.Errorf("strings cannot be subtracted") - } else if lhsTag == "!!int" && rhsTag == "!!int" { - format, lhsNum, err := parseInt64(lhs.Value) - if err != nil { - return err - } - _, rhsNum, err := parseInt64(rhs.Value) - if err != nil { - return err - } - result := lhsNum - rhsNum - target.Node.Tag = lhs.Tag - target.Node.Value = fmt.Sprintf(format, result) - } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { - lhsNum, err := strconv.ParseFloat(lhs.Value, 64) - if err != nil { - return err - } - rhsNum, err := strconv.ParseFloat(rhs.Value, 64) - if err != nil { - return err - } - result := lhsNum - rhsNum - if lhsIsCustom { - target.Node.Tag = lhs.Tag - } else { - target.Node.Tag = "!!float" - } - target.Node.Value = fmt.Sprintf("%v", result) - } else { - return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) - } - - return nil -} - -func subtractDateTime(layout string, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { - var durationStr string - if strings.HasPrefix(rhs.Value, "-") { - durationStr = rhs.Value[1:] - } else { - durationStr = "-" + rhs.Value - } - duration, err := time.ParseDuration(durationStr) - - if err != nil { - return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) - } - - currentTime, err := parseDateTime(layout, lhs.Value) - if err != nil { - return err - } - - newTime := currentTime.Add(duration) - target.Node.Value = newTime.Format(layout) - return nil -} diff --git a/external/yq/pkg/yqlib/operator_subtract_test.go b/external/yq/pkg/yqlib/operator_subtract_test.go deleted file mode 100755 index aaaa91f..0000000 --- a/external/yq/pkg/yqlib/operator_subtract_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var subtractOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `{}`, - expression: "(.a - .b) as $x", - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - description: "Array subtraction", - expression: `[1,2] - [2,3]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n", - }, - }, - { - skipDoc: true, - expression: `[2,1,2,2] - [2,3]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n", - }, - }, - { - description: "Array subtraction with nested array", - expression: `[[1], 1, 2] - [[1], 3]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- 2\n", - }, - }, - { - skipDoc: true, - expression: `[[1], 1, [[[2]]]] - [[1], [[[3]]]]`, - expected: []string{ - "D0, P[], (!!seq)::- 1\n- - - - 2\n", - }, - }, - { - description: "Array subtraction with nested object", - subdescription: `Note that order of the keys does not matter`, - document: `[{a: b, c: d}, {a: b}]`, - expression: `. - [{"c": "d", "a": "b"}]`, - expected: []string{ - "D0, P[], (!!seq)::[{a: b}]\n", - }, - }, - { - skipDoc: true, - document: `[{a: [1], c: d}, {a: [2], c: d}, {a: b}]`, - expression: `. - [{"c": "d", "a": [1]}]`, - expected: []string{ - "D0, P[], (!!seq)::[{a: [2], c: d}, {a: b}]\n", - }, - }, - { - description: "Number subtraction - float", - subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", - document: `{a: 3, b: 4.5}`, - expression: `.a = .a - .b`, - expected: []string{ - "D0, P[], (doc)::{a: -1.5, b: 4.5}\n", - }, - }, - { - description: "Number subtraction - float", - subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", - document: `{a: 3, b: 4.5}`, - expression: `.a = .a - .b`, - expected: []string{ - "D0, P[], (doc)::{a: -1.5, b: 4.5}\n", - }, - }, - { - description: "Number subtraction - int", - subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", - document: `{a: 3, b: 4}`, - expression: `.a = .a - .b`, - expected: []string{ - "D0, P[], (doc)::{a: -1, b: 4}\n", - }, - }, - { - description: "Decrement numbers", - document: `{a: 3, b: 5}`, - expression: `.[] -= 1`, - expected: []string{ - "D0, P[], (doc)::{a: 2, b: 4}\n", - }, - }, - { - description: "Date subtraction", - subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: 2021-01-01T03:10:00Z`, - expression: `.a -= "3h10m"`, - expected: []string{ - "D0, P[], (doc)::a: 2021-01-01T00:00:00Z\n", - }, - }, - { - description: "Date subtraction - only date", - skipDoc: true, - document: `a: 2021-01-01`, - expression: `.a -= "24h"`, - expected: []string{ - "D0, P[], (doc)::a: 2020-12-31T00:00:00Z\n", - }, - }, - { - description: "Date subtraction - custom format", - subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: Saturday, 15-Dec-01 at 6:00AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, - expected: []string{ - "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n", - }, - }, - { - skipDoc: true, - description: "Date subtraction - custom format", - subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - document: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`, - expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, - expected: []string{ - "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n", - }, - }, - { - description: "Custom types: that are really numbers", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse 2\nb: !goat 1", - expression: `.a -= .b`, - expected: []string{ - "D0, P[], (doc)::a: !horse 1\nb: !goat 1\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really floats", - subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - document: "a: !horse 2.5\nb: !goat 1.5", - expression: `.a - .b`, - expected: []string{ - "D0, P[a], (!horse)::1\n", - }, - }, - { - skipDoc: true, - description: "Custom types: that are really maps", - document: `[!horse {a: b, c: d}, !goat {a: b}]`, - expression: `. - [{"c": "d", "a": "b"}]`, - expected: []string{ - "D0, P[], (!!seq)::[!goat {a: b}]\n", - }, - }, -} - -func TestSubtractOperatorScenarios(t *testing.T) { - for _, tt := range subtractOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "subtract", subtractOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_tag.go b/external/yq/pkg/yqlib/operator_tag.go deleted file mode 100755 index 59b07d8..0000000 --- a/external/yq/pkg/yqlib/operator_tag.go +++ /dev/null @@ -1,63 +0,0 @@ -package yqlib - -import ( - "container/list" - - yaml "gopkg.in/yaml.v3" -) - -func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("AssignTagOperator: %v") - tag := "" - - if !expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - tag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - - if err != nil { - return Context{}, err - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - log.Debugf("Setting tag of : %v", candidate.GetKey()) - if expressionNode.Operation.UpdateAssign { - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) - if err != nil { - return Context{}, err - } - - if rhs.MatchingNodes.Front() != nil { - tag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - } - unwrapDoc(candidate.Node).Tag = tag - } - - return context, nil -} - -func getTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("GetTagOperator") - - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - node := &yaml.Node{Kind: yaml.ScalarNode, Value: unwrapDoc(candidate.Node).Tag, Tag: "!!str"} - result := candidate.CreateReplacement(node) - results.PushBack(result) - } - - return context.ChildContext(results), nil -} diff --git a/external/yq/pkg/yqlib/operator_tag_test.go b/external/yq/pkg/yqlib/operator_tag_test.go deleted file mode 100755 index 0466e5f..0000000 --- a/external/yq/pkg/yqlib/operator_tag_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var tagOperatorScenarios = []expressionScenario{ - { - description: "Get tag", - document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, - expression: `.. | tag`, - expected: []string{ - "D0, P[], (!!str)::!!map\n", - "D0, P[a], (!!str)::!!str\n", - "D0, P[b], (!!str)::!!int\n", - "D0, P[c], (!!str)::!!float\n", - "D0, P[e], (!!str)::!!bool\n", - "D0, P[f], (!!str)::!!seq\n", - }, - }, - { - description: "type is an alias for tag", - document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, - expression: `.. | type`, - expected: []string{ - "D0, P[], (!!str)::!!map\n", - "D0, P[a], (!!str)::!!str\n", - "D0, P[b], (!!str)::!!int\n", - "D0, P[c], (!!str)::!!float\n", - "D0, P[e], (!!str)::!!bool\n", - "D0, P[f], (!!str)::!!seq\n", - }, - }, - { - skipDoc: true, - document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, - expression: `tag`, - expected: []string{ - "D0, P[], (!!str)::!!map\n", - }, - }, - { - skipDoc: true, - document: `32`, - expression: `. tag= "!!str"`, - expected: []string{ - "D0, P[], (doc)::\"32\"\n", - }, - }, - { - description: "Set custom tag", - document: `{a: str}`, - expression: `.a tag = "!!mikefarah"`, - expected: []string{ - "D0, P[], (doc)::{a: !!mikefarah str}\n", - }, - }, - { - skipDoc: true, - description: "Set custom type", - document: `{a: str}`, - expression: `.a type = "!!mikefarah"`, - expected: []string{ - "D0, P[], (doc)::{a: !!mikefarah str}\n", - }, - }, - { - description: "Find numbers and convert them to strings", - document: `{a: cat, b: 5, c: 3.2, e: true}`, - expression: `(.. | select(tag == "!!int")) tag= "!!str"`, - expected: []string{ - "D0, P[], (!!map)::{a: cat, b: \"5\", c: 3.2, e: true}\n", - }, - }, - { - skipDoc: true, - document: `{a: "!!frog", b: "!!customTag"}`, - expression: `.[] tag |= .`, - expected: []string{ - "D0, P[], (doc)::{a: !!frog \"!!frog\", b: !!customTag \"!!customTag\"}\n", - }, - }, -} - -func TestTagOperatorScenarios(t *testing.T) { - for _, tt := range tagOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "tag", tagOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_traverse_path.go b/external/yq/pkg/yqlib/operator_traverse_path.go deleted file mode 100755 index 2a8bf27..0000000 --- a/external/yq/pkg/yqlib/operator_traverse_path.go +++ /dev/null @@ -1,331 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "github.com/elliotchance/orderedmap" - yaml "gopkg.in/yaml.v3" -) - -type traversePreferences struct { - DontFollowAlias bool - IncludeMapKeys bool - DontAutoCreate bool // by default, we automatically create entries on the fly. - DontIncludeMapValues bool - OptionalTraverse bool // e.g. .adf? -} - -func splat(context Context, prefs traversePreferences) (Context, error) { - return traverseNodesWithArrayIndices(context, make([]*yaml.Node, 0), prefs) -} - -func traversePathOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- traversePathOperator") - var matches = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - newNodes, err := traverse(context, el.Value.(*CandidateNode), expressionNode.Operation) - if err != nil { - return Context{}, err - } - matches.PushBackList(newNodes) - } - - return context.ChildContext(matches), nil -} - -func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) { - log.Debug("Traversing %v", NodeToString(matchingNode)) - value := matchingNode.Node - - if value.Tag == "!!null" && operation.Value != "[]" { - log.Debugf("Guessing kind") - // we must have added this automatically, lets guess what it should be now - switch operation.Value.(type) { - case int, int64: - log.Debugf("probably an array") - value.Kind = yaml.SequenceNode - default: - log.Debugf("probably a map") - value.Kind = yaml.MappingNode - } - value.Tag = "" - } - - switch value.Kind { - case yaml.MappingNode: - log.Debug("its a map with %v entries", len(value.Content)/2) - return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false) - - case yaml.SequenceNode: - log.Debug("its a sequence of %v things!", len(value.Content)) - return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences)) - - case yaml.AliasNode: - log.Debug("its an alias!") - matchingNode.Node = matchingNode.Node.Alias - return traverse(context, matchingNode, operation) - case yaml.DocumentNode: - log.Debug("digging into doc node") - - return traverse(context, matchingNode.CreateChildInMap(nil, matchingNode.Node.Content[0]), operation) - default: - return list.New(), nil - } -} - -func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - //lhs may update the variable context, we should pass that into the RHS - // BUT we still return the original context back (see jq) - // https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|... - - if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType { - return sliceArrayOperator(d, context, expressionNode.RHS.RHS) - } - - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - - // rhs is a collect expression that will yield indexes to retrieve of the arrays - - rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - prefs := traversePreferences{} - - if expressionNode.Operation.Preferences != nil { - prefs = expressionNode.Operation.Preferences.(traversePreferences) - } - var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content - - //now we traverse the result of the lhs against the indices we found - result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs) - if err != nil { - return Context{}, err - } - return context.ChildContext(result.MatchingNodes), nil -} - -func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*yaml.Node, prefs traversePreferences) (Context, error) { - var matchingNodeMap = list.New() - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - newNodes, err := traverseArrayIndices(context, candidate, indicesToTraverse, prefs) - if err != nil { - return Context{}, err - } - matchingNodeMap.PushBackList(newNodes) - } - - return context.ChildContext(matchingNodeMap), nil -} - -func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse - node := matchingNode.Node - if node.Tag == "!!null" { - log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array") - // auto vivification - node.Tag = "" - node.Kind = yaml.SequenceNode - //check that the indices are numeric, if not, then we should create an object - if len(indicesToTraverse) != 0 && indicesToTraverse[0].Tag != "!!int" { - node.Kind = yaml.MappingNode - } - } - - if node.Kind == yaml.AliasNode { - matchingNode.Node = node.Alias - return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs) - } else if node.Kind == yaml.SequenceNode { - return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs) - } else if node.Kind == yaml.MappingNode { - return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs) - } else if node.Kind == yaml.DocumentNode { - return traverseArrayIndices(context, matchingNode.CreateChildInMap(nil, matchingNode.Node.Content[0]), indicesToTraverse, prefs) - } - log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag) - return list.New(), nil -} - -func traverseMapWithIndices(context Context, candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) { - if len(indices) == 0 { - return traverseMap(context, candidate, createStringScalarNode(""), prefs, true) - } - - var matchingNodeMap = list.New() - - for _, indexNode := range indices { - log.Debug("traverseMapWithIndices: %v", indexNode.Value) - newNodes, err := traverseMap(context, candidate, indexNode, prefs, false) - if err != nil { - return nil, err - } - matchingNodeMap.PushBackList(newNodes) - } - - return matchingNodeMap, nil -} - -func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) { - log.Debug("traverseArrayWithIndices") - var newMatches = list.New() - node := unwrapDoc(candidate.Node) - if len(indices) == 0 { - log.Debug("splatting") - var index int - for index = 0; index < len(node.Content); index = index + 1 { - newMatches.PushBack(candidate.CreateChildInArray(index, node.Content[index])) - } - return newMatches, nil - - } - - for _, indexNode := range indices { - log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value) - index, err := parseInt(indexNode.Value) - if err != nil && prefs.OptionalTraverse { - continue - } - if err != nil { - return nil, fmt.Errorf("cannot index array with '%v' (%w)", indexNode.Value, err) - } - indexToUse := index - contentLength := len(node.Content) - for contentLength <= index { - if contentLength == 0 { - // default to nice yaml formating - node.Style = 0 - } - - node.Content = append(node.Content, &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}) - contentLength = len(node.Content) - } - - if indexToUse < 0 { - indexToUse = contentLength + indexToUse - } - - if indexToUse < 0 { - return nil, fmt.Errorf("index [%v] out of range, array size is %v", index, contentLength) - } - - newMatches.PushBack(candidate.CreateChildInArray(index, node.Content[indexToUse])) - } - return newMatches, nil -} - -func keyMatches(key *yaml.Node, wantedKey string) bool { - return matchKey(key.Value, wantedKey) -} - -func traverseMap(context Context, matchingNode *CandidateNode, keyNode *yaml.Node, prefs traversePreferences, splat bool) (*list.List, error) { - var newMatches = orderedmap.NewOrderedMap() - err := doTraverseMap(newMatches, matchingNode, keyNode.Value, prefs, splat) - - if err != nil { - return nil, err - } - - if !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 { - //no matches, create one automagically - valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"} - - node := matchingNode.Node - - if len(node.Content) == 0 { - node.Style = 0 - } - - node.Content = append(node.Content, keyNode, valueNode) - - if prefs.IncludeMapKeys { - log.Debug("including key") - candidateNode := matchingNode.CreateChildInMap(keyNode, keyNode) - candidateNode.IsMapKey = true - newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode) - } - if !prefs.DontIncludeMapValues { - log.Debug("including value") - candidateNode := matchingNode.CreateChildInMap(keyNode, valueNode) - newMatches.Set(candidateNode.GetKey(), candidateNode) - } - } - - results := list.New() - i := 0 - for el := newMatches.Front(); el != nil; el = el.Next() { - results.PushBack(el.Value) - i++ - } - return results, nil -} - -func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { - // value.Content is a concatenated array of key, value, - // so keys are in the even indexes, values in odd. - // merge aliases are defined first, but we only want to traverse them - // if we don't find a match directly on this node first. - - node := candidate.Node - - var contents = node.Content - for index := 0; index < len(contents); index = index + 2 { - key := contents[index] - value := contents[index+1] - - log.Debug("checking %v (%v)", key.Value, key.Tag) - //skip the 'merge' tag, find a direct match first - if key.Tag == "!!merge" && !prefs.DontFollowAlias { - log.Debug("Merge anchor") - err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat) - if err != nil { - return err - } - } else if splat || keyMatches(key, wantedKey) { - log.Debug("MATCHED") - if prefs.IncludeMapKeys { - log.Debug("including key") - candidateNode := candidate.CreateChildInMap(key, key) - candidateNode.IsMapKey = true - newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode) - } - if !prefs.DontIncludeMapValues { - log.Debug("including value") - candidateNode := candidate.CreateChildInMap(key, value) - newMatches.Set(candidateNode.GetKey(), candidateNode) - } - } - } - - return nil -} - -func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error { - switch value.Kind { - case yaml.AliasNode: - if value.Alias.Kind != yaml.MappingNode { - return fmt.Errorf("can only use merge anchors with maps (!!map), but got %v", value.Alias.Tag) - } - candidateNode := originalCandidate.CreateReplacement(value.Alias) - return doTraverseMap(newMatches, candidateNode, wantedKey, prefs, splat) - case yaml.SequenceNode: - for _, childValue := range value.Content { - err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, prefs, splat) - if err != nil { - return err - } - } - } - return nil -} - -func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) { - log.Debug("operation Value %v", operation.Value) - indices := []*yaml.Node{{Value: operation.StringValue}} - return traverseArrayWithIndices(candidate, indices, prefs) -} diff --git a/external/yq/pkg/yqlib/operator_traverse_path_test.go b/external/yq/pkg/yqlib/operator_traverse_path_test.go deleted file mode 100755 index 56b3cc9..0000000 --- a/external/yq/pkg/yqlib/operator_traverse_path_test.go +++ /dev/null @@ -1,561 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var mergeDocSample = ` -foo: &foo - a: foo_a - thing: foo_thing - c: foo_c - -bar: &bar - b: bar_b - thing: bar_thing - c: bar_c - -foobarList: - b: foobarList_b - <<: [*foo,*bar] - c: foobarList_c - -foobar: - c: foobar_c - <<: *foo - thing: foobar_thing -` - -// cannot use merge anchors with arrays -var badAliasSample = ` -_common: &common-docker-file - - FROM ubuntu:18.04 - -steps: - <<: *common-docker-file -` - -var traversePathOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `[[1]]`, - expression: `.[0][0]`, - expected: []string{ - "D0, P[0 0], (!!int)::1\n", - }, - }, - { - skipDoc: true, - expression: `.cat["12"] = "things"`, - expected: []string{ - "D0, P[], ()::cat:\n \"12\": things\n", - }, - }, - { - skipDoc: true, - document: `blah: {}`, - expression: `.blah.cat = "cool"`, - expected: []string{ - "D0, P[], (doc)::blah:\n cat: cool\n", - }, - }, - { - skipDoc: true, - document: `blah: []`, - expression: `.blah.0 = "cool"`, - expected: []string{ - "D0, P[], (doc)::blah:\n - cool\n", - }, - }, - { - skipDoc: true, - document: `b: cat`, - expression: ".b\n", - expected: []string{ - "D0, P[b], (!!str)::cat\n", - }, - }, - { - skipDoc: true, - document: `[[[1]]]`, - expression: `.[0][0][0]`, - expected: []string{ - "D0, P[0 0 0], (!!int)::1\n", - }, - }, - { - skipDoc: true, - expression: `.["cat"] = "thing"`, - expected: []string{ - "D0, P[], ()::cat: thing\n", - }, - }, - { - description: "Simple map navigation", - document: `{a: {b: apple}}`, - expression: `.a`, - expected: []string{ - "D0, P[a], (!!map)::{b: apple}\n", - }, - }, - { - description: "Splat", - subdescription: "Often used to pipe children into other operators", - document: `[{b: apple}, {c: banana}]`, - expression: `.[]`, - expected: []string{ - "D0, P[0], (!!map)::{b: apple}\n", - "D0, P[1], (!!map)::{c: banana}\n", - }, - }, - { - description: "Optional Splat", - subdescription: "Just like splat, but won't error if you run it against scalars", - document: `"cat"`, - expression: `.[]`, - expected: []string{}, - }, - { - description: "Special characters", - subdescription: "Use quotes with brackets around path elements with special characters", - document: `{"{}": frog}`, - expression: `.["{}"]`, - expected: []string{ - "D0, P[{}], (!!str)::frog\n", - }, - }, - { - description: "Nested special characters", - document: `a: {"key.withdots": {"another.key": apple}}`, - expression: `.a["key.withdots"]["another.key"]`, - expected: []string{ - "D0, P[a key.withdots another.key], (!!str)::apple\n", - }, - }, - { - description: "Keys with spaces", - subdescription: "Use quotes with brackets around path elements with special characters", - document: `{"red rabbit": frog}`, - expression: `.["red rabbit"]`, - expected: []string{ - "D0, P[red rabbit], (!!str)::frog\n", - }, - }, - { - skipDoc: true, - document: `{"flying fox": frog}`, - expression: `.["flying fox"]`, - expected: []string{ - "D0, P[flying fox], (!!str)::frog\n", - }, - }, - { - skipDoc: true, - document: `c: dog`, - expression: `.[.a.b] as $x`, - expected: []string{ - "D0, P[], (doc)::c: dog\n", - }, - }, - { - description: "Dynamic keys", - subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`, - document: `{b: apple, apple: crispy yum, banana: soft yum}`, - expression: `.[.b]`, - expected: []string{ - "D0, P[apple], (!!str)::crispy yum\n", - }, - }, - { - skipDoc: true, - document: `{b: apple, fruit: {apple: yum, banana: smooth}}`, - expression: `.fruit[.b]`, - expected: []string{ - "D0, P[fruit apple], (!!str)::yum\n", - }, - }, - { - description: "Children don't exist", - subdescription: "Nodes are added dynamically while traversing", - document: `{c: banana}`, - expression: `.a.b`, - expected: []string{ - "D0, P[a b], (!!null)::null\n", - }, - }, - { - description: "Optional identifier", - subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected", - document: `[1,2,3]`, - expression: `.a?`, - expected: []string{}, - }, - { - skipDoc: true, - document: `[[1,2,3], {a: frog}]`, - expression: `.[] | .["a"]?`, - expected: []string{"D0, P[1 a], (!!str)::frog\n"}, - }, - { - skipDoc: true, - document: ``, - expression: `.[1].a`, - expected: []string{ - "D0, P[1 a], (!!null)::null\n", - }, - }, - { - skipDoc: true, - document: `{}`, - expression: `.a[1]`, - expected: []string{ - "D0, P[a 1], (!!null)::null\n", - }, - }, - { - description: "Wildcard matching", - document: `{a: {cat: apple, mad: things}}`, - expression: `.a."*a*"`, - expected: []string{ - "D0, P[a cat], (!!str)::apple\n", - "D0, P[a mad], (!!str)::things\n", - }, - }, - { - skipDoc: true, - document: `{a: {cat: {b: 3}, mad: {b: 4}, fad: {c: t}}}`, - expression: `.a."*a*".b`, - expected: []string{ - "D0, P[a cat b], (!!int)::3\n", - "D0, P[a mad b], (!!int)::4\n", - "D0, P[a fad b], (!!null)::null\n", - }, - }, - { - skipDoc: true, - document: `{a: {cat: apple, mad: things}}`, - expression: `.a | (.cat, .mad)`, - expected: []string{ - "D0, P[a cat], (!!str)::apple\n", - "D0, P[a mad], (!!str)::things\n", - }, - }, - { - skipDoc: true, - document: `{a: {cat: apple, mad: things}}`, - expression: `.a | (.cat, .mad, .fad)`, - expected: []string{ - "D0, P[a cat], (!!str)::apple\n", - "D0, P[a mad], (!!str)::things\n", - "D0, P[a fad], (!!null)::null\n", - }, - }, - { - skipDoc: true, - document: `{a: {cat: apple, mad: things}}`, - expression: `.a | (.cat, .mad, .fad) | select( (. == null) | not)`, - expected: []string{ - "D0, P[a cat], (!!str)::apple\n", - "D0, P[a mad], (!!str)::things\n", - }, - }, - { - description: "Aliases", - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `.b`, - expected: []string{ - "D0, P[b], (alias)::*cat\n", - }, - }, - { - description: "Traversing aliases with splat", - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `.b[]`, - expected: []string{ - "D0, P[b c], (!!str)::frog\n", - }, - }, - { - skipDoc: true, - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `.b[]`, - expected: []string{ - "D0, P[b c], (!!str)::frog\n", - }, - }, - { - description: "Traversing aliases explicitly", - document: `{a: &cat {c: frog}, b: *cat}`, - expression: `.b.c`, - expected: []string{ - "D0, P[b c], (!!str)::frog\n", - }, - }, - { - description: "Traversing arrays by index", - document: `[1,2,3]`, - expression: `.[0]`, - expected: []string{ - "D0, P[0], (!!int)::1\n", - }, - }, - { - description: "Traversing nested arrays by index", - dontFormatInputForDoc: true, - document: `[[], [cat]]`, - expression: `.[1][0]`, - expected: []string{ - "D0, P[1 0], (!!str)::cat\n", - }, - }, - { - description: "Maps with numeric keys", - document: `{2: cat}`, - expression: `.[2]`, - expected: []string{ - "D0, P[2], (!!str)::cat\n", - }, - }, - { - description: "Maps with non existing numeric keys", - document: `{a: b}`, - expression: `.[0]`, - expected: []string{ - "D0, P[0], (!!null)::null\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobar`, - expected: []string{ - "D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n", - }, - }, - { - description: "Traversing merge anchors", - document: mergeDocSample, - expression: `.foobar.a`, - expected: []string{ - "D0, P[foobar a], (!!str)::foo_a\n", - }, - }, - { - description: "Traversing merge anchors with override", - document: mergeDocSample, - expression: `.foobar.c`, - expected: []string{ - "D0, P[foobar c], (!!str)::foo_c\n", - }, - }, - { - description: "Traversing merge anchors with local override", - document: mergeDocSample, - expression: `.foobar.thing`, - expected: []string{ - "D0, P[foobar thing], (!!str)::foobar_thing\n", - }, - }, - { - description: "Splatting merge anchors", - document: mergeDocSample, - expression: `.foobar[]`, - expected: []string{ - "D0, P[foobar c], (!!str)::foo_c\n", - "D0, P[foobar a], (!!str)::foo_a\n", - "D0, P[foobar thing], (!!str)::foobar_thing\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobar[]`, - expected: []string{ - "D0, P[foobar c], (!!str)::foo_c\n", - "D0, P[foobar a], (!!str)::foo_a\n", - "D0, P[foobar thing], (!!str)::foobar_thing\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList`, - expected: []string{ - "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList.a`, - expected: []string{ - "D0, P[foobarList a], (!!str)::foo_a\n", - }, - }, - { - description: "Traversing merge anchor lists", - subdescription: "Note that the later merge anchors override previous", - document: mergeDocSample, - expression: `.foobarList.thing`, - expected: []string{ - "D0, P[foobarList thing], (!!str)::bar_thing\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList.c`, - expected: []string{ - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList.b`, - expected: []string{ - "D0, P[foobarList b], (!!str)::bar_b\n", - }, - }, - { - description: "Splatting merge anchor lists", - document: mergeDocSample, - expression: `.foobarList[]`, - expected: []string{ - "D0, P[foobarList b], (!!str)::bar_b\n", - "D0, P[foobarList a], (!!str)::foo_a\n", - "D0, P[foobarList thing], (!!str)::bar_thing\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, - { - skipDoc: true, - document: mergeDocSample, - expression: `.foobarList[]`, - expected: []string{ - "D0, P[foobarList b], (!!str)::bar_b\n", - "D0, P[foobarList a], (!!str)::foo_a\n", - "D0, P[foobarList thing], (!!str)::bar_thing\n", - "D0, P[foobarList c], (!!str)::foobarList_c\n", - }, - }, - { - skipDoc: true, - document: `[a,b,c]`, - expression: `.[]`, - expected: []string{ - "D0, P[0], (!!str)::a\n", - "D0, P[1], (!!str)::b\n", - "D0, P[2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `[a,b,c]`, - expression: `[]`, - expected: []string{ - "D0, P[], (!!seq)::[]\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[0]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - }, - }, - { - description: "Select multiple indices", - document: `{a: [a,b,c]}`, - expression: `.a[0, 2]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - "D0, P[a 2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[0, 2]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - "D0, P[a 2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[-1]`, - expected: []string{ - "D0, P[a -1], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[-1]`, - expected: []string{ - "D0, P[a -1], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[-2]`, - expected: []string{ - "D0, P[a -2], (!!str)::b\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[-2]`, - expected: []string{ - "D0, P[a -2], (!!str)::b\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - "D0, P[a 1], (!!str)::b\n", - "D0, P[a 2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a[]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - "D0, P[a 1], (!!str)::b\n", - "D0, P[a 2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: `{a: [a,b,c]}`, - expression: `.a | .[]`, - expected: []string{ - "D0, P[a 0], (!!str)::a\n", - "D0, P[a 1], (!!str)::b\n", - "D0, P[a 2], (!!str)::c\n", - }, - }, - { - skipDoc: true, - document: badAliasSample, - expression: ".steps[]", - expectedError: "can only use merge anchors with maps (!!map), but got !!seq", - }, -} - -func TestTraversePathOperatorScenarios(t *testing.T) { - for _, tt := range traversePathOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "traverse-read", traversePathOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_union.go b/external/yq/pkg/yqlib/operator_union.go deleted file mode 100755 index 310d827..0000000 --- a/external/yq/pkg/yqlib/operator_union.go +++ /dev/null @@ -1,44 +0,0 @@ -package yqlib - -import "container/list" - -func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debug("unionOperator") - log.Debug("context: %v", NodesToString(context.MatchingNodes)) - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - log.Debug("lhs: %v", NodesToString(lhs.MatchingNodes)) - log.Debug("rhs input: %v", NodesToString(context.MatchingNodes)) - log.Debug("rhs: %v", expressionNode.RHS.Operation.toString()) - rhs, err := d.GetMatchingNodes(context, expressionNode.RHS) - - if err != nil { - return Context{}, err - } - log.Debug("lhs: %v", lhs.ToString()) - log.Debug("rhs: %v", rhs.ToString()) - - results := lhs.ChildContext(list.New()) - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - node := el.Value.(*CandidateNode) - results.MatchingNodes.PushBack(node) - } - - // this can happen when both expressions modify the context - // instead of creating their own. - /// (.foo = "bar"), (.thing = "cat") - if rhs.MatchingNodes != lhs.MatchingNodes { - - for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() { - node := el.Value.(*CandidateNode) - log.Debug("processing %v", NodeToString(node)) - - results.MatchingNodes.PushBack(node) - } - } - log.Debug("and lets print it out") - log.Debug("all together: %v", results.ToString()) - return results, nil -} diff --git a/external/yq/pkg/yqlib/operator_union_test.go b/external/yq/pkg/yqlib/operator_union_test.go deleted file mode 100755 index ce12f19..0000000 --- a/external/yq/pkg/yqlib/operator_union_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var unionOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: "{}", - expression: `(.a, .b.c) as $x`, - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - skipDoc: true, - description: "clone test", - expression: `"abc" as $a | [$a, "cat"]`, - expected: []string{ - "D0, P[], (!!seq)::- abc\n- cat\n", - }, - }, - { - skipDoc: true, - expression: `(.foo = "bar"), (.toe = "jam")`, - expected: []string{ - "D0, P[], ()::foo: bar\ntoe: jam\n", - }, - }, - { - description: "Combine scalars", - expression: `1, true, "cat"`, - expected: []string{ - "D0, P[], (!!int)::1\n", - "D0, P[], (!!bool)::true\n", - "D0, P[], (!!str)::cat\n", - }, - }, - { - description: "Combine selected paths", - document: `{a: fieldA, b: fieldB, c: fieldC}`, - expression: `.a, .c`, - expected: []string{ - "D0, P[a], (!!str)::fieldA\n", - "D0, P[c], (!!str)::fieldC\n", - }, - }, -} - -func TestUnionOperatorScenarios(t *testing.T) { - for _, tt := range unionOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "union", unionOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_unique.go b/external/yq/pkg/yqlib/operator_unique.go deleted file mode 100755 index a1c5216..0000000 --- a/external/yq/pkg/yqlib/operator_unique.go +++ /dev/null @@ -1,64 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "github.com/elliotchance/orderedmap" - yaml "gopkg.in/yaml.v3" -) - -func unique(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} - uniqueByExpression := &ExpressionNode{Operation: &Operation{OperationType: uniqueByOpType}, RHS: selfExpression} - return uniqueBy(d, context, uniqueByExpression) - -} - -func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - - log.Debugf("-- uniqueBy Operator") - var results = list.New() - - for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - candidateNode := unwrapDoc(candidate.Node) - - if candidateNode.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("Only arrays are supported for unique") - } - - var newMatches = orderedmap.NewOrderedMap() - for _, node := range candidateNode.Content { - child := &CandidateNode{Node: node} - rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.RHS) - - if err != nil { - return Context{}, err - } - - keyValue := "null" - - if rhs.MatchingNodes.Len() > 0 { - first := rhs.MatchingNodes.Front() - keyCandidate := first.Value.(*CandidateNode) - keyValue = keyCandidate.Node.Value - } - - _, exists := newMatches.Get(keyValue) - - if !exists { - newMatches.Set(keyValue, child.Node) - } - } - resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} - for el := newMatches.Front(); el != nil; el = el.Next() { - resultNode.Content = append(resultNode.Content, el.Value.(*yaml.Node)) - } - - results.PushBack(candidate.CreateReplacementWithDocWrappers(resultNode)) - } - - return context.ChildContext(results), nil - -} diff --git a/external/yq/pkg/yqlib/operator_unique_test.go b/external/yq/pkg/yqlib/operator_unique_test.go deleted file mode 100755 index f0c4453..0000000 --- a/external/yq/pkg/yqlib/operator_unique_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var uniqueOperatorScenarios = []expressionScenario{ - { - description: "Unique array of scalars (string/numbers)", - subdescription: "Note that unique maintains the original order of the array.", - document: `[2,1,3,2]`, - expression: `unique`, - expected: []string{ - "D0, P[], (!!seq)::- 2\n- 1\n- 3\n", - }, - }, - { - description: "Unique nulls", - subdescription: "Unique works on the node value, so it considers different representations of nulls to be different", - document: `[~,null, ~, null]`, - expression: `unique`, - expected: []string{ - "D0, P[], (!!seq)::- ~\n- null\n", - }, - }, - { - description: "Unique all nulls", - subdescription: "Run against the node tag to unique all the nulls", - document: `[~,null, ~, null]`, - expression: `unique_by(tag)`, - expected: []string{ - "D0, P[], (!!seq)::- ~\n", - }, - }, - { - description: "Unique array object fields", - document: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`, - expression: `unique_by(.name)`, - expected: []string{ - "D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {name: billy, pet: dog}\n", - }, - }, - { - skipDoc: true, - document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, - expression: `unique_by(.name)`, - expected: []string{ - "D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {pet: fish}\n", - }, - }, - { - skipDoc: true, - document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, - expression: `unique_by(.cat.dog)`, - expected: []string{ - "D0, P[], (!!seq)::- {name: harry, pet: cat}\n", - }, - }, - { - skipDoc: true, - document: "# abc\n[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]\n# xyz", - expression: `unique_by(.name)`, - expected: []string{ - "D0, P[], (!!seq)::# abc\n- {name: harry, pet: cat}\n- {pet: fish}\n# xyz\n", - }, - }, -} - -func TestUniqueOperatorScenarios(t *testing.T) { - for _, tt := range uniqueOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "unique", uniqueOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_value.go b/external/yq/pkg/yqlib/operator_value.go deleted file mode 100755 index 6c70b1f..0000000 --- a/external/yq/pkg/yqlib/operator_value.go +++ /dev/null @@ -1,6 +0,0 @@ -package yqlib - -func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value) - return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil -} diff --git a/external/yq/pkg/yqlib/operator_value_test.go b/external/yq/pkg/yqlib/operator_value_test.go deleted file mode 100755 index 8153480..0000000 --- a/external/yq/pkg/yqlib/operator_value_test.go +++ /dev/null @@ -1,169 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var valueOperatorScenarios = []expressionScenario{ - { - document: ``, - expression: `1`, - expected: []string{ - "D0, P[], (!!int)::1\n", - }, - }, - { - document: ``, - expression: `0x9f`, - expected: []string{ - "D0, P[], (!!int)::0x9f\n", - }, - }, - { - document: ``, - expression: `0x1A`, - expected: []string{ - "D0, P[], (!!int)::0x1A\n", - }, - }, - { - document: ``, - expression: `0x1A + 2`, - expected: []string{ - "D0, P[], (!!int)::0x1C\n", - }, - }, - { - document: ``, - expression: `0x12 * 2`, - expected: []string{ - "D0, P[], (!!int)::0x24\n", - }, - }, - { - document: ``, - expression: `0xF - 1`, - expected: []string{ - "D0, P[], (!!int)::0xE\n", - }, - }, - { - document: ``, - expression: `12`, - expected: []string{ - "D0, P[], (!!int)::12\n", - }, - }, - { - document: ``, - expression: `12 + 2`, - expected: []string{ - "D0, P[], (!!int)::14\n", - }, - }, - { - document: ``, - expression: `12 * 2`, - expected: []string{ - "D0, P[], (!!int)::24\n", - }, - }, - { - document: ``, - expression: `12 - 2`, - expected: []string{ - "D0, P[], (!!int)::10\n", - }, - }, - { - document: ``, - expression: `0X12`, - expected: []string{ - "D0, P[], (!!int)::0X12\n", - }, - }, - { - document: ``, - expression: `-1`, - expected: []string{ - "D0, P[], (!!int)::-1\n", - }, - }, { - document: ``, - expression: `1.2`, - expected: []string{ - "D0, P[], (!!float)::1.2\n", - }, - }, { - document: ``, - expression: `-5.2e11`, - expected: []string{ - "D0, P[], (!!float)::-5.2e11\n", - }, - }, { - document: ``, - expression: `5e-10`, - expected: []string{ - "D0, P[], (!!float)::5e-10\n", - }, - }, - { - document: ``, - expression: `"cat"`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - document: ``, - expression: `"frog jumps"`, - expected: []string{ - "D0, P[], (!!str)::frog jumps\n", - }, - }, - { - document: ``, - expression: `"1.3"`, - expected: []string{ - "D0, P[], (!!str)::1.3\n", - }, - }, { - document: ``, - expression: `"true"`, - expected: []string{ - "D0, P[], (!!str)::true\n", - }, - }, { - document: ``, - expression: `true`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, { - document: ``, - expression: `false`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, - }, - { - document: ``, - expression: `Null`, - expected: []string{ - "D0, P[], (!!null)::Null\n", - }, - }, - { - document: ``, - expression: `~`, - expected: []string{ - "D0, P[], (!!null)::~\n", - }, - }, -} - -func TestValueOperatorScenarios(t *testing.T) { - for _, tt := range valueOperatorScenarios { - testScenario(t, &tt) - } -} diff --git a/external/yq/pkg/yqlib/operator_variables.go b/external/yq/pkg/yqlib/operator_variables.go deleted file mode 100755 index 5550bde..0000000 --- a/external/yq/pkg/yqlib/operator_variables.go +++ /dev/null @@ -1,42 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" -) - -func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - variableName := expressionNode.Operation.StringValue - log.Debug("getVariableOperator %v", variableName) - result := context.GetVariable(variableName) - if result == nil { - result = list.New() - } - return context.ChildContext(result), nil -} - -type assignVarPreferences struct { - IsReference bool -} - -func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.LHS) - if err != nil { - return Context{}, err - } - if expressionNode.RHS.Operation.OperationType.Type != "GET_VARIABLE" { - return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo") - } - variableName := expressionNode.RHS.Operation.StringValue - - prefs := expressionNode.Operation.Preferences.(assignVarPreferences) - - var variableValue *list.List - if prefs.IsReference { - variableValue = lhs.MatchingNodes - } else { - variableValue = lhs.DeepClone().MatchingNodes - } - context.SetVariable(variableName, variableValue) - return context, nil -} diff --git a/external/yq/pkg/yqlib/operator_variables_test.go b/external/yq/pkg/yqlib/operator_variables_test.go deleted file mode 100755 index 7ab9ee3..0000000 --- a/external/yq/pkg/yqlib/operator_variables_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package yqlib - -import ( - "testing" -) - -var variableOperatorScenarios = []expressionScenario{ - { - skipDoc: true, - document: `{}`, - expression: `.a.b as $foo`, - expected: []string{ - "D0, P[], (doc)::{}\n", - }, - }, - { - document: "a: [cat]", - skipDoc: true, - expression: "(.[] | {.name: .}) as $item", - expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`, - }, - { - description: "Single value variable", - document: `a: cat`, - expression: `.a as $foo | $foo`, - expected: []string{ - "D0, P[a], (!!str)::cat\n", - }, - }, - { - description: "Multi value variable", - document: `[cat, dog]`, - expression: `.[] as $foo | $foo`, - expected: []string{ - "D0, P[0], (!!str)::cat\n", - "D0, P[1], (!!str)::dog\n", - }, - }, - { - description: "Using variables as a lookup", - subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)", - document: `{"posts": [{"title": "First post", "author": "anon"}, - {"title": "A well-written article", "author": "person1"}], - "realnames": {"anon": "Anonymous Coward", - "person1": "Person McPherson"}}`, - expression: `.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}`, - expected: []string{ - "D0, P[], (!!map)::title: \"First post\"\nauthor: \"Anonymous Coward\"\n", - "D0, P[], (!!map)::title: \"A well-written article\"\nauthor: \"Person McPherson\"\n", - }, - }, - { - description: "Using variables to swap values", - document: "a: a_value\nb: b_value", - expression: `.a as $x | .b as $y | .b = $x | .a = $y`, - expected: []string{ - "D0, P[], (doc)::a: b_value\nb: a_value\n", - }, - }, - { - description: "Use ref to reference a path repeatedly", - subdescription: "Note: You may find the `with` operator more useful.", - document: `a: {b: thing, c: something}`, - expression: `.a.b ref $x | $x = "new" | $x style="double"`, - expected: []string{ - "D0, P[], (doc)::a: {b: \"new\", c: something}\n", - }, - }, -} - -func TestVariableOperatorScenarios(t *testing.T) { - for _, tt := range variableOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "variable-operators", variableOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operator_with.go b/external/yq/pkg/yqlib/operator_with.go deleted file mode 100755 index fd76ec6..0000000 --- a/external/yq/pkg/yqlib/operator_with.go +++ /dev/null @@ -1,34 +0,0 @@ -package yqlib - -import "fmt" - -func withOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - log.Debugf("-- withOperator") - // with(path, exp) - - if expressionNode.RHS.Operation.OperationType != blockOpType { - return Context{}, fmt.Errorf("with must be given a block (;), got %v instead", expressionNode.RHS.Operation.OperationType.Type) - } - - pathExp := expressionNode.RHS.LHS - - updateContext, err := d.GetMatchingNodes(context, pathExp) - - if err != nil { - return Context{}, err - } - - updateExp := expressionNode.RHS.RHS - - for el := updateContext.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - _, err = d.GetMatchingNodes(updateContext.SingleChildContext(candidate), updateExp) - if err != nil { - return Context{}, err - } - - } - - return context, nil - -} diff --git a/external/yq/pkg/yqlib/operator_with_test.go b/external/yq/pkg/yqlib/operator_with_test.go deleted file mode 100755 index 04f184c..0000000 --- a/external/yq/pkg/yqlib/operator_with_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package yqlib - -import "testing" - -var withOperatorScenarios = []expressionScenario{ - { - description: "Update and style", - document: `a: {deeply: {nested: value}}`, - expression: `with(.a.deeply.nested; . = "newValue" | . style="single")`, - expected: []string{ - "D0, P[], (doc)::a: {deeply: {nested: 'newValue'}}\n", - }, - }, - { - description: "Update multiple deeply nested properties", - document: `a: {deeply: {nested: value, other: thing}}`, - expression: `with(.a.deeply; .nested = "newValue" | .other= "newThing")`, - expected: []string{ - "D0, P[], (doc)::a: {deeply: {nested: newValue, other: newThing}}\n", - }, - }, - { - description: "Update array elements relatively", - subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.", - document: `myArray: [{a: apple},{a: banana}]`, - expression: `with(.myArray[]; .b = .a + " yum")`, - expected: []string{ - "D0, P[], (doc)::myArray: [{a: apple, b: apple yum}, {a: banana, b: banana yum}]\n", - }, - }, - { - description: "Update array elements relatively +=", - skipDoc: true, - subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.", - document: `myArray: [{a: apple},{a: banana}]`, - expression: `with(.myArray[]; .a += .a)`, - expected: []string{ - "D0, P[], (doc)::myArray: [{a: appleapple}, {a: bananabanana}]\n", - }, - }, -} - -func TestWithOperatorScenarios(t *testing.T) { - for _, tt := range withOperatorScenarios { - testScenario(t, &tt) - } - documentOperatorScenarios(t, "with", withOperatorScenarios) -} diff --git a/external/yq/pkg/yqlib/operators.go b/external/yq/pkg/yqlib/operators.go deleted file mode 100755 index 00f45d4..0000000 --- a/external/yq/pkg/yqlib/operators.go +++ /dev/null @@ -1,215 +0,0 @@ -package yqlib - -import ( - "container/list" - "fmt" - - "github.com/jinzhu/copier" - logging "gopkg.in/op/go-logging.v1" - "gopkg.in/yaml.v3" -) - -type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) - -type compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode - -func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) { - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - - // tricky logic when we are running *= with flags. - // we have an op like: .a *=nc .b - // which should roughly translate to .a =c .a *nc .b - // note that the 'n' flag only applies to the multiple op, not the assignment - // but the clobber flag applies to both! - - prefs := assignPreferences{} - switch typedPref := expressionNode.Operation.Preferences.(type) { - case assignPreferences: - prefs = typedPref - case multiplyPreferences: - prefs.ClobberCustomTags = typedPref.AssignPrefs.ClobberCustomTags - } - - assignmentOp := &Operation{OperationType: assignOpType, Preferences: prefs} - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - clone, err := candidate.Copy() - if err != nil { - return Context{}, err - } - valueCopyExp := &ExpressionNode{Operation: &Operation{OperationType: valueOpType, CandidateNode: clone}} - - valueExpression := &ExpressionNode{Operation: &Operation{OperationType: valueOpType, CandidateNode: candidate}} - - assignmentOpNode := &ExpressionNode{Operation: assignmentOp, LHS: valueExpression, RHS: calculation(valueCopyExp, expressionNode.RHS)} - - _, err = d.GetMatchingNodes(context, assignmentOpNode) - if err != nil { - return Context{}, err - } - } - return context, nil -} - -func unwrapDoc(node *yaml.Node) *yaml.Node { - if node.Kind == yaml.DocumentNode { - return node.Content[0] - } - return node -} - -func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - context.MatchingNodes = list.New() - return context, nil -} - -type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) - -func resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, prefs crossFunctionPreferences, rhsExp *ExpressionNode, results *list.List) error { - - if prefs.LhsResultValue != nil { - result, err := prefs.LhsResultValue(lhsCandidate) - if err != nil { - return err - } else if result != nil { - results.PushBack(result) - return nil - } - } - - rhs, err := d.GetMatchingNodes(context, rhsExp) - if err != nil { - return err - } - - if prefs.CalcWhenEmpty && rhs.MatchingNodes.Len() == 0 { - resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, nil) - if err != nil { - return err - } - if resultCandidate != nil { - results.PushBack(resultCandidate) - } - return nil - } - - for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() { - rhsCandidate := rightEl.Value.(*CandidateNode) - if !log.IsEnabledFor(logging.DEBUG) { - log.Debugf("Applying lhs: %v, rhsCandidate, %v", NodeToString(lhsCandidate), NodeToString(rhsCandidate)) - } - resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate) - if err != nil { - return err - } - if resultCandidate != nil { - results.PushBack(resultCandidate) - } - } - return nil -} - -type crossFunctionPreferences struct { - CalcWhenEmpty bool - // if this returns a result node, - // we wont bother calculating the RHS - LhsResultValue func(*CandidateNode) (*CandidateNode, error) - Calculation crossFunctionCalculation -} - -func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) { - var results = list.New() - lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) - if err != nil { - return Context{}, err - } - log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len()) - - if prefs.CalcWhenEmpty && lhs.MatchingNodes.Len() == 0 { - err := resultsForRHS(d, context, nil, prefs, expressionNode.RHS, results) - if err != nil { - return Context{}, err - } - } - - for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { - lhsCandidate := el.Value.(*CandidateNode) - - err = resultsForRHS(d, context, lhsCandidate, prefs, expressionNode.RHS, results) - if err != nil { - return Context{}, err - } - - } - return context.ChildContext(results), nil -} - -func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) { - prefs := crossFunctionPreferences{CalcWhenEmpty: calcWhenEmpty, Calculation: calculation} - return crossFunctionWithPrefs(d, context, expressionNode, prefs) -} - -func crossFunctionWithPrefs(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) { - var results = list.New() - - var evaluateAllTogether = true - for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { - evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether - if !evaluateAllTogether { - break - } - } - - if evaluateAllTogether { - log.Debug("crossFunction evaluateAllTogether!") - return doCrossFunc(d, context, expressionNode, prefs) - } - - log.Debug("crossFunction evaluate apart!") - - for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { - innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, prefs) - if err != nil { - return Context{}, err - } - results.PushBackList(innerResults.MatchingNodes) - } - - return context.ChildContext(results), nil -} - -func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode { - valString := "true" - if !value { - valString = "false" - } - node := &yaml.Node{Kind: yaml.ScalarNode, Value: valString, Tag: "!!bool"} - return owner.CreateReplacement(node) -} - -func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode { - if len(path) == 0 { - return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} - } else if len(path) == 1 { - lastPrefs := traversePrefs - if targetKey { - err := copier.Copy(&lastPrefs, traversePrefs) - if err != nil { - panic(err) - } - lastPrefs.IncludeMapKeys = true - lastPrefs.DontIncludeMapValues = true - } - return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} - } - - return &ExpressionNode{ - Operation: &Operation{OperationType: shortPipeOpType}, - LHS: createTraversalTree(path[0:1], traversePrefs, false), - RHS: createTraversalTree(path[1:], traversePrefs, targetKey), - } -} diff --git a/external/yq/pkg/yqlib/operators_test.go b/external/yq/pkg/yqlib/operators_test.go deleted file mode 100755 index 7975789..0000000 --- a/external/yq/pkg/yqlib/operators_test.go +++ /dev/null @@ -1,360 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "container/list" - "fmt" - "io" - "os" - "sort" - "strings" - "testing" - "time" - - "github.com/mikefarah/yq/v4/test" - logging "gopkg.in/op/go-logging.v1" - yaml "gopkg.in/yaml.v3" -) - -type expressionScenario struct { - description string - subdescription string - environmentVariables map[string]string - document string - document2 string - expression string - expected []string - skipDoc bool - expectedError string - dontFormatInputForDoc bool // dont format input doc for documentation generation -} - -func TestMain(m *testing.M) { - logging.SetLevel(logging.ERROR, "") - Now = func() time.Time { - return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) - } - code := m.Run() - os.Exit(code) -} - -func NewSimpleYamlPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { - prefs := NewDefaultYamlPreferences() - prefs.PrintDocSeparators = printDocSeparators - prefs.UnwrapScalar = unwrapScalar - return NewPrinter(NewYamlEncoder(indent, colorsEnabled, prefs), NewSinglePrinterWriter(writer)) -} - -func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { - reader := bufio.NewReader(strings.NewReader(content)) - - return readDocuments(reader, fakefilename, fakeFileIndex, NewYamlDecoder(ConfiguredYamlPreferences)) -} - -func testScenario(t *testing.T, s *expressionScenario) { - var err error - node, err := getExpressionParser().ParseExpression(s.expression) - if err != nil { - t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) - return - } - inputs := list.New() - - if s.document != "" { - inputs, err = readDocument(s.document, "sample.yml", 0) - - if err != nil { - t.Error(err, s.document, s.expression) - return - } - - if s.document2 != "" { - moreInputs, err := readDocument(s.document2, "another.yml", 1) - if err != nil { - t.Error(err, s.document2, s.expression) - return - } - inputs.PushBackList(moreInputs) - } - } else { - candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode}, - FileIndex: 0, - } - inputs.PushBack(candidateNode) - - } - - for name, value := range s.environmentVariables { - os.Setenv(name, value) - } - - context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node) - - if s.expectedError != "" { - if err == nil { - t.Errorf("Expected error '%v' but it worked!", s.expectedError) - } else { - test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) - } - return - } - - if err != nil { - t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression)) - return - } - test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) -} - -func resultToString(t *testing.T, n *CandidateNode) string { - var valueBuffer bytes.Buffer - printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), YamlOutputFormat, true, false, 4, true) - - err := printer.PrintResults(n.AsList()) - if err != nil { - t.Error(err) - return "" - } - - tag := n.Node.Tag - if n.Node.Kind == yaml.DocumentNode { - tag = "doc" - } else if n.Node.Kind == yaml.AliasNode { - tag = "alias" - } - return fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.Document, n.Path, tag, valueBuffer.String()) -} - -func resultsToString(t *testing.T, results *list.List) []string { - var pretty = make([]string, 0) - - for el := results.Front(); el != nil; el = el.Next() { - n := el.Value.(*CandidateNode) - - output := resultToString(t, n) - pretty = append(pretty, output) - } - return pretty -} - -func writeOrPanic(w *bufio.Writer, text string) { - _, err := w.WriteString(text) - if err != nil { - panic(err) - } -} - -func copySnippet(source string, out *os.File) error { - _, err := os.Stat(source) - if os.IsNotExist(err) { - return nil - } - in, err := os.Open(source) - if err != nil { - return err - } - defer safelyCloseFile(in) - _, err = io.Copy(out, in) - return err -} - -func formatYaml(yaml string, filename string) string { - var output bytes.Buffer - printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) - - node, err := getExpressionParser().ParseExpression(".. style= \"\"") - if err != nil { - panic(err) - } - streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - return output.String() -} - -type documentScenarioFunc func(t *testing.T, writer *bufio.Writer, scenario interface{}) - -func documentScenarios(t *testing.T, folder string, title string, scenarios []interface{}, documentScenario documentScenarioFunc) { - f, err := os.Create(fmt.Sprintf("doc/%v/%v.md", folder, title)) - - if err != nil { - t.Error(err) - return - } - defer f.Close() - - source := fmt.Sprintf("doc/%v/headers/%v.md", folder, title) - err = copySnippet(source, f) - if err != nil { - t.Error(err) - return - } - - err = copySnippet("doc/notification-snippet.md", f) - if err != nil { - t.Error(err) - return - } - - w := bufio.NewWriter(f) - writeOrPanic(w, "\n") - - for _, s := range scenarios { - documentScenario(t, w, s) - } - w.Flush() -} - -func documentOperatorScenarios(t *testing.T, title string, scenarios []expressionScenario) { - genericScenarios := make([]interface{}, len(scenarios)) - for i, s := range scenarios { - genericScenarios[i] = s - } - - documentScenarios(t, "operators", title, genericScenarios, documentOperatorScenario) -} - -func documentOperatorScenario(t *testing.T, w *bufio.Writer, i interface{}) { - s := i.(expressionScenario) - - if s.skipDoc { - return - } - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - formattedDoc, formattedDoc2 := documentInput(w, s) - - writeOrPanic(w, "will output\n") - - documentOutput(t, w, s, formattedDoc, formattedDoc2) -} - -func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { - formattedDoc := "" - formattedDoc2 := "" - command := "" - - envCommand := "" - - envKeys := make([]string, 0, len(s.environmentVariables)) - for k := range s.environmentVariables { - envKeys = append(envKeys, k) - } - sort.Strings(envKeys) - - for _, name := range envKeys { - value := s.environmentVariables[name] - if envCommand == "" { - envCommand = fmt.Sprintf("%v=\"%v\" ", name, value) - } else { - envCommand = fmt.Sprintf("%v %v=\"%v\" ", envCommand, name, value) - } - os.Setenv(name, value) - } - - if s.document != "" { - if s.dontFormatInputForDoc { - formattedDoc = s.document + "\n" - } else { - formattedDoc = formatYaml(s.document, "sample.yml") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc)) - - files := "sample.yml" - - if s.document2 != "" { - if s.dontFormatInputForDoc { - formattedDoc2 = s.document2 + "\n" - } else { - formattedDoc2 = formatYaml(s.document2, "another.yml") - } - - writeOrPanic(w, "And another sample another.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc2)) - files = "sample.yml another.yml" - command = "eval-all " - } - - writeOrPanic(w, "then\n") - - if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, s.expression, files)) - } else { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) - } - } else { - writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) - } - return formattedDoc, formattedDoc2 -} - -func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) { - var output bytes.Buffer - var err error - printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) - - node, err := getExpressionParser().ParseExpression(s.expression) - if err != nil { - t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) - return - } - - inputs := list.New() - - if s.document != "" { - - inputs, err = readDocument(formattedDoc, "sample.yml", 0) - if err != nil { - t.Error(err, s.document, s.expression) - return - } - if s.document2 != "" { - moreInputs, err := readDocument(formattedDoc2, "another.yml", 1) - if err != nil { - t.Error(err, s.document, s.expression) - return - } - inputs.PushBackList(moreInputs) - } - } else { - candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode}, - FileIndex: 0, - } - inputs.PushBack(candidateNode) - - } - - context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node) - - if s.expectedError != "" && err != nil { - writeOrPanic(w, fmt.Sprintf("```bash\nError: %v\n```\n\n", err.Error())) - return - } else if err != nil { - t.Error(err, s.expression) - return - } - - err = printer.PrintResults(context.MatchingNodes) - if err != nil { - t.Error(err, s.expression) - } - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String())) -} diff --git a/external/yq/pkg/yqlib/printer.go b/external/yq/pkg/yqlib/printer.go deleted file mode 100755 index aef082d..0000000 --- a/external/yq/pkg/yqlib/printer.go +++ /dev/null @@ -1,167 +0,0 @@ -package yqlib - -import ( - "bufio" - "container/list" - "fmt" - "io" - "regexp" - - yaml "gopkg.in/yaml.v3" -) - -type Printer interface { - PrintResults(matchingNodes *list.List) error - PrintedAnything() bool - //e.g. when given a front-matter doc, like jekyll - SetAppendix(reader io.Reader) -} - -type PrinterOutputFormat uint32 - -const ( - YamlOutputFormat = 1 << iota - JSONOutputFormat - PropsOutputFormat - CSVOutputFormat - TSVOutputFormat - XMLOutputFormat - Base64OutputFormat -) - -func OutputFormatFromString(format string) (PrinterOutputFormat, error) { - switch format { - case "yaml", "y": - return YamlOutputFormat, nil - case "json", "j": - return JSONOutputFormat, nil - case "props", "p": - return PropsOutputFormat, nil - case "csv", "c": - return CSVOutputFormat, nil - case "tsv", "t": - return TSVOutputFormat, nil - case "xml", "x": - return XMLOutputFormat, nil - default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format) - } -} - -type resultsPrinter struct { - encoder Encoder - printerWriter PrinterWriter - firstTimePrinting bool - previousDocIndex uint - previousFileIndex int - printedMatches bool - treeNavigator DataTreeNavigator - appendixReader io.Reader -} - -func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer { - return &resultsPrinter{ - encoder: encoder, - printerWriter: printerWriter, - firstTimePrinting: true, - treeNavigator: NewDataTreeNavigator(), - } -} - -func (p *resultsPrinter) SetAppendix(reader io.Reader) { - p.appendixReader = reader -} - -func (p *resultsPrinter) PrintedAnything() bool { - return p.printedMatches -} - -func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error { - p.printedMatches = p.printedMatches || (node.Tag != "!!null" && - (node.Tag != "!!bool" || node.Value != "false")) - return p.encoder.Encode(writer, node) -} - -func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { - log.Debug("PrintResults for %v matches", matchingNodes.Len()) - - if matchingNodes.Len() == 0 { - log.Debug("no matching results, nothing to print") - return nil - } - - if !p.encoder.CanHandleAliases() { - explodeOp := Operation{OperationType: explodeOpType} - explodeNode := ExpressionNode{Operation: &explodeOp} - context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode) - if err != nil { - return err - } - matchingNodes = context.MatchingNodes - } - - if p.firstTimePrinting { - node := matchingNodes.Front().Value.(*CandidateNode) - p.previousDocIndex = node.Document - p.previousFileIndex = node.FileIndex - p.firstTimePrinting = false - } - - for el := matchingNodes.Front(); el != nil; el = el.Next() { - - mappedDoc := el.Value.(*CandidateNode) - log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document) - log.Debug("%v", NodeToString(mappedDoc)) - writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) - if errorWriting != nil { - return errorWriting - } - - commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`) - commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) - - if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator { - if err := p.encoder.PrintDocumentSeparator(writer); err != nil { - return err - } - } - - if err := p.encoder.PrintLeadingContent(writer, mappedDoc.LeadingContent); err != nil { - return err - } - - if err := p.printNode(mappedDoc.Node, writer); err != nil { - return err - } - - if err := p.encoder.PrintLeadingContent(writer, mappedDoc.TrailingContent); err != nil { - return err - } - - p.previousDocIndex = mappedDoc.Document - if err := writer.Flush(); err != nil { - return err - } - log.Debugf("done printing results") - } - - // what happens if I remove output format check? - if p.appendixReader != nil { - writer, err := p.printerWriter.GetWriter(nil) - if err != nil { - return err - } - - log.Debug("Piping appendix reader...") - betterReader := bufio.NewReader(p.appendixReader) - _, err = io.Copy(writer, betterReader) - if err != nil { - return err - } - if err := writer.Flush(); err != nil { - return err - } - } - - return nil -} diff --git a/external/yq/pkg/yqlib/printer_test.go b/external/yq/pkg/yqlib/printer_test.go deleted file mode 100755 index cc7b3db..0000000 --- a/external/yq/pkg/yqlib/printer_test.go +++ /dev/null @@ -1,338 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "container/list" - "strings" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -var multiDocSample = `a: banana ---- -a: apple ---- -a: coconut -` - -var multiDocSampleLeadingExpected = `# go cats ---- -a: banana ---- -a: apple ---- -# cool -a: coconut -` - -func nodeToList(candidate *CandidateNode) *list.List { - elMap := list.New() - elMap.PushBack(candidate) - return elMap -} - -func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - el := inputs.Front() - sample1 := nodeToList(el.Value.(*CandidateNode)) - - el = el.Next() - sample2 := nodeToList(el.Value.(*CandidateNode)) - - el = el.Next() - sample3 := nodeToList(el.Value.(*CandidateNode)) - - err = printer.PrintResults(sample1) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample2) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample3) - if err != nil { - panic(err) - } - - writer.Flush() - test.AssertResult(t, multiDocSample, output.String()) -} - -func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - el := inputs.Front() - el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n" - sample1 := nodeToList(el.Value.(*CandidateNode)) - - el = el.Next() - el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n" - sample2 := nodeToList(el.Value.(*CandidateNode)) - - el = el.Next() - el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n# cool\n" - sample3 := nodeToList(el.Value.(*CandidateNode)) - - err = printer.PrintResults(sample1) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample2) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample3) - if err != nil { - panic(err) - } - - writer.Flush() - - test.AssertResult(t, multiDocSampleLeadingExpected, output.String()) -} - -func TestPrinterMultipleFilesInSequence(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - el := inputs.Front() - elNode := el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 0 - sample1 := nodeToList(elNode) - - el = el.Next() - elNode = el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 1 - sample2 := nodeToList(elNode) - - el = el.Next() - elNode = el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 2 - sample3 := nodeToList(elNode) - - err = printer.PrintResults(sample1) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample2) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample3) - if err != nil { - panic(err) - } - - writer.Flush() - test.AssertResult(t, multiDocSample, output.String()) -} - -func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - el := inputs.Front() - elNode := el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 0 - elNode.LeadingContent = "# go cats\n$yqDocSeperator$\n" - sample1 := nodeToList(elNode) - - el = el.Next() - elNode = el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 1 - elNode.LeadingContent = "$yqDocSeperator$\n" - sample2 := nodeToList(elNode) - - el = el.Next() - elNode = el.Value.(*CandidateNode) - elNode.Document = 0 - elNode.FileIndex = 2 - elNode.LeadingContent = "$yqDocSeperator$\n# cool\n" - sample3 := nodeToList(elNode) - - err = printer.PrintResults(sample1) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample2) - if err != nil { - panic(err) - } - - err = printer.PrintResults(sample3) - if err != nil { - panic(err) - } - - writer.Flush() - test.AssertResult(t, multiDocSampleLeadingExpected, output.String()) -} - -func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - err = printer.PrintResults(inputs) - if err != nil { - panic(err) - } - - writer.Flush() - test.AssertResult(t, multiDocSample, output.String()) -} - -func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n" - - err = printer.PrintResults(inputs) - if err != nil { - panic(err) - } - - writer.Flush() - expected := `# go cats ---- -a: banana ---- -a: apple ---- -a: coconut -` - test.AssertResult(t, expected, output.String()) -} - -func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n" - err = printer.PrintResults(inputs) - if err != nil { - panic(err) - } - - writer.Flush() - expected := `--- -a: banana ---- -a: apple ---- -a: coconut -` - test.AssertResult(t, expected, output.String()) -} - -func TestPrinterScalarWithLeadingCont(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - - node, err := getExpressionParser().ParseExpression(".a") - if err != nil { - panic(err) - } - streamEvaluator := NewStreamEvaluator() - _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - writer.Flush() - expected := `banana ---- -apple ---- -coconut -` - test.AssertResult(t, expected, output.String()) -} - -func TestPrinterMultipleDocsJson(t *testing.T) { - var output bytes.Buffer - var writer = bufio.NewWriter(&output) - // note printDocSeparators is true, it should still not print document separators - // when outputing JSON. - printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer)) - - inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) - if err != nil { - panic(err) - } - - inputs.Front().Value.(*CandidateNode).LeadingContent = "# ignore this\n" - - err = printer.PrintResults(inputs) - if err != nil { - panic(err) - } - - expected := `{"a":"banana"} -{"a":"apple"} -{"a":"coconut"} -` - - writer.Flush() - test.AssertResult(t, expected, output.String()) -} diff --git a/external/yq/pkg/yqlib/printer_writer.go b/external/yq/pkg/yqlib/printer_writer.go deleted file mode 100755 index daa1078..0000000 --- a/external/yq/pkg/yqlib/printer_writer.go +++ /dev/null @@ -1,85 +0,0 @@ -package yqlib - -import ( - "bufio" - "fmt" - "io" - "os" - "regexp" - - "gopkg.in/yaml.v3" -) - -type PrinterWriter interface { - GetWriter(node *CandidateNode) (*bufio.Writer, error) -} - -type singlePrinterWriter struct { - bufferedWriter *bufio.Writer -} - -func NewSinglePrinterWriter(writer io.Writer) PrinterWriter { - return &singlePrinterWriter{ - bufferedWriter: bufio.NewWriter(writer), - } -} - -func (sp *singlePrinterWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) { - return sp.bufferedWriter, nil -} - -type multiPrintWriter struct { - treeNavigator DataTreeNavigator - nameExpression *ExpressionNode - extension string - index int -} - -func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter { - extension := "yml" - - switch format { - case JSONOutputFormat: - extension = "json" - case PropsOutputFormat: - extension = "properties" - } - - return &multiPrintWriter{ - nameExpression: expression, - extension: extension, - treeNavigator: NewDataTreeNavigator(), - index: 0, - } -} - -func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) { - name := "" - - indexVariableNode := yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)} - indexVariableCandidate := CandidateNode{Node: &indexVariableNode} - - context := Context{MatchingNodes: node.AsList()} - context.SetVariable("index", indexVariableCandidate.AsList()) - result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression) - if err != nil { - return nil, err - } - if result.MatchingNodes.Len() > 0 { - name = result.MatchingNodes.Front().Value.(*CandidateNode).Node.Value - } - var extensionRegexp = regexp.MustCompile(`\.[a-zA-Z0-9]+$`) - if !extensionRegexp.MatchString(name) { - name = fmt.Sprintf("%v.%v", name, sp.extension) - } - - f, err := os.Create(name) - - if err != nil { - return nil, err - } - sp.index = sp.index + 1 - - return bufio.NewWriter(f), nil - -} diff --git a/external/yq/pkg/yqlib/properties_test.go b/external/yq/pkg/yqlib/properties_test.go deleted file mode 100755 index 324a46d..0000000 --- a/external/yq/pkg/yqlib/properties_test.go +++ /dev/null @@ -1,350 +0,0 @@ -package yqlib - -import ( - "bufio" - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -const propertiesWithCommentsOnMap = `this.thing = hi hi -# important notes -# about this value -this.value = cool -` - -const expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi - -# important notes -# about this value -this.value = cool -` - -const expectedPropertiesWithCommentsOnMapYaml = `this: - thing: hi hi - # important notes - # about this value - value: cool -` - -const propertiesWithCommentInArray = ` -this.array.0 = cat -# important notes -# about dogs -this.array.1 = dog -` - -const expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat - -# important notes -# about dogs -this.array.1 = dog -` - -const expectedPropertiesWithCommentInArrayYaml = `this: - array: - - cat - # important notes - # about dogs - - dog -` - -const samplePropertiesYaml = `# block comments come through -person: # neither do comments on maps - name: Mike Wazowski # comments on values appear - pets: - - cat # comments on array values appear - food: [pizza] # comments on arrays do not -emptyArray: [] -emptyMap: [] -` - -const expectedPropertiesUnwrapped = `# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -` - -const expectedPropertiesWrapped = `# block comments come through -# comments on values appear -person.name = "Mike Wazowski" - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -` - -const expectedUpdatedProperties = `# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = dog -person.food.0 = pizza -` - -const expectedDecodedYaml = `person: - # block comments come through - # comments on values appear - name: Mike Wazowski - pets: - # comments on array values appear - - cat - food: - - pizza -` - -const expectedPropertiesNoComments = `person.name = Mike Wazowski -person.pets.0 = cat -person.food.0 = pizza -` - -const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through -# comments on values appear -person.name = Mike Wazowski - -# comments on array values appear -person.pets.0 = cat -person.food.0 = pizza -emptyArray = -emptyMap = -` - -var propertyScenarios = []formatScenario{ - { - description: "Encode properties", - subdescription: "Note that empty arrays and maps are not encoded by default.", - input: samplePropertiesYaml, - expected: expectedPropertiesUnwrapped, - }, - { - description: "Encode properties: scalar encapsulation", - subdescription: "Note that string values with blank characters in them are encapsulated with double quotes", - input: samplePropertiesYaml, - expected: expectedPropertiesWrapped, - scenarioType: "encode-wrapped", - }, - { - description: "Encode properties: no comments", - input: samplePropertiesYaml, - expected: expectedPropertiesNoComments, - expression: `... comments = ""`, - }, - { - description: "Encode properties: include empty maps and arrays", - subdescription: "Use a yq expression to set the empty maps and sequences to your desired value.", - expression: `(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ""`, - input: samplePropertiesYaml, - expected: expectedPropertiesWithEmptyMapsAndArrays, - }, - { - description: "Decode properties", - input: expectedPropertiesUnwrapped, - expected: expectedDecodedYaml, - scenarioType: "decode", - }, - { - description: "Decode properties - array should be a map", - subdescription: "If you have a numeric map key in your property files, use array_to_map to convert them to maps.", - input: `things.10 = mike`, - expression: `.things |= array_to_map`, - expected: "things:\n 10: mike\n", - scenarioType: "decode", - }, - { - description: "does not expand automatically", - skipDoc: true, - input: "mike = ${dontExpand} this", - expected: "mike: ${dontExpand} this\n", - scenarioType: "decode", - }, - { - description: "Roundtrip", - input: expectedPropertiesUnwrapped, - expression: `.person.pets.0 = "dog"`, - expected: expectedUpdatedProperties, - scenarioType: "roundtrip", - }, - { - skipDoc: true, - description: "comments on arrays roundtrip", - input: propertiesWithCommentInArray, - expected: expectedPropertiesWithCommentInArrayProps, - scenarioType: "roundtrip", - }, - { - skipDoc: true, - description: "comments on arrays decode", - input: propertiesWithCommentInArray, - expected: expectedPropertiesWithCommentInArrayYaml, - scenarioType: "decode", - }, - { - skipDoc: true, - description: "comments on map roundtrip", - input: propertiesWithCommentsOnMap, - expected: expectedPropertiesWithCommentsOnMapProps, - scenarioType: "roundtrip", - }, - { - skipDoc: true, - description: "comments on map decode", - input: propertiesWithCommentsOnMap, - expected: expectedPropertiesWithCommentsOnMapYaml, - scenarioType: "decode", - }, - { - description: "Empty doc", - skipDoc: true, - input: "", - expected: "", - scenarioType: "decode", - }, -} - -func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props '%v' sample.yml\n```\n", expression)) - } else { - writeOrPanic(w, "```bash\nyq -o=props sample.yml\n```\n") - } - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)))) -} - -func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props --unwrapScalar=false '%v' sample.yml\n```\n", expression)) - } else { - writeOrPanic(w, "```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\n") - } - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)))) -} - -func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.properties file of:\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=props '%v' sample.properties\n```\n", expression)) - } else { - writeOrPanic(w, "```bash\nyq -p=props sample.properties\n```\n") - } - - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences)))) -} - -func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.properties file of:\n") - writeOrPanic(w, fmt.Sprintf("```properties\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - - expression := s.expression - if expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=props -o=props '%v' sample.properties\n```\n", expression)) - } else { - writeOrPanic(w, "```bash\nyq -p=props -o=props sample.properties\n```\n") - } - - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)))) -} - -func documentPropertyScenario(t *testing.T, w *bufio.Writer, i interface{}) { - s := i.(formatScenario) - if s.skipDoc { - return - } - switch s.scenarioType { - case "": - documentUnwrappedEncodePropertyScenario(w, s) - case "decode": - documentDecodePropertyScenario(w, s) - case "encode-wrapped": - documentWrappedEncodePropertyScenario(w, s) - case "roundtrip": - documentRoundTripPropertyScenario(w, s) - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func TestPropertyScenarios(t *testing.T) { - for _, s := range propertyScenarios { - switch s.scenarioType { - case "": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(true)), s.description) - case "decode": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "encode-wrapped": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(false)), s.description) - case "roundtrip": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(true)), s.description) - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } - } - genericScenarios := make([]interface{}, len(propertyScenarios)) - for i, s := range propertyScenarios { - genericScenarios[i] = s - } - documentScenarios(t, "usage", "properties", genericScenarios, documentPropertyScenario) -} diff --git a/external/yq/pkg/yqlib/stream_evaluator.go b/external/yq/pkg/yqlib/stream_evaluator.go deleted file mode 100755 index ef78ca9..0000000 --- a/external/yq/pkg/yqlib/stream_evaluator.go +++ /dev/null @@ -1,119 +0,0 @@ -package yqlib - -import ( - "container/list" - "errors" - "fmt" - "io" - "os" - - yaml "gopkg.in/yaml.v3" -) - -// A yaml expression evaluator that runs the expression multiple times for each given yaml document. -// Uses less memory than loading all documents and running the expression once, but this cannot process -// cross document expressions. -type StreamEvaluator interface { - Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) - EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error - EvaluateNew(expression string, printer Printer) error -} - -type streamEvaluator struct { - treeNavigator DataTreeNavigator - fileIndex int -} - -func NewStreamEvaluator() StreamEvaluator { - return &streamEvaluator{treeNavigator: NewDataTreeNavigator()} -} - -func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error { - node, err := ExpressionParser.ParseExpression(expression) - if err != nil { - return err - } - candidateNode := &CandidateNode{ - Document: 0, - Filename: "", - Node: &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, - FileIndex: 0, - } - inputList := list.New() - inputList.PushBack(candidateNode) - - result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node) - if errorParsing != nil { - return errorParsing - } - return printer.PrintResults(result.MatchingNodes) -} - -func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { - var totalProcessDocs uint - node, err := ExpressionParser.ParseExpression(expression) - if err != nil { - return err - } - - for _, filename := range filenames { - reader, err := readStream(filename) - - if err != nil { - return err - } - processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder) - if err != nil { - return err - } - totalProcessDocs = totalProcessDocs + processedDocs - - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } - } - - if totalProcessDocs == 0 { - // problem is I've already slurped the leading content sadface - return s.EvaluateNew(expression, printer) - } - - return nil -} - -func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) { - - var currentIndex uint - err := decoder.Init(reader) - if err != nil { - return 0, err - } - for { - candidateNode, errorReading := decoder.Decode() - - if errors.Is(errorReading, io.EOF) { - s.fileIndex = s.fileIndex + 1 - return currentIndex, nil - } else if errorReading != nil { - return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading) - } - candidateNode.Document = currentIndex - candidateNode.Filename = filename - candidateNode.FileIndex = s.fileIndex - - inputList := list.New() - inputList.PushBack(candidateNode) - - result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node) - if errorParsing != nil { - return currentIndex, errorParsing - } - err := printer.PrintResults(result.MatchingNodes) - - if err != nil { - return currentIndex, err - } - currentIndex = currentIndex + 1 - } -} diff --git a/external/yq/pkg/yqlib/string_evaluator.go b/external/yq/pkg/yqlib/string_evaluator.go deleted file mode 100755 index 59d451c..0000000 --- a/external/yq/pkg/yqlib/string_evaluator.go +++ /dev/null @@ -1,73 +0,0 @@ -package yqlib - -import ( - "bufio" - "bytes" - "container/list" - "errors" - "fmt" - "io" - "strings" -) - -type StringEvaluator interface { - Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) -} - -type stringEvaluator struct { - treeNavigator DataTreeNavigator - fileIndex int -} - -func NewStringEvaluator() StringEvaluator { - return &stringEvaluator{ - treeNavigator: NewDataTreeNavigator(), - } -} - -func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) { - - // Use bytes.Buffer for output of string - out := new(bytes.Buffer) - printer := NewPrinter(encoder, NewSinglePrinterWriter(out)) - - InitExpressionParser() - node, err := ExpressionParser.ParseExpression(expression) - if err != nil { - return "", err - } - - reader := bufio.NewReader(strings.NewReader(input)) - - var currentIndex uint - err = decoder.Init(reader) - if err != nil { - return "", err - } - for { - candidateNode, errorReading := decoder.Decode() - - if errors.Is(errorReading, io.EOF) { - s.fileIndex = s.fileIndex + 1 - return out.String(), nil - } else if errorReading != nil { - return "", fmt.Errorf("bad input '%v': %w", input, errorReading) - } - candidateNode.Document = currentIndex - candidateNode.FileIndex = s.fileIndex - - inputList := list.New() - inputList.PushBack(candidateNode) - - result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node) - if errorParsing != nil { - return "", errorParsing - } - err = printer.PrintResults(result.MatchingNodes) - - if err != nil { - return "", err - } - currentIndex = currentIndex + 1 - } -} diff --git a/external/yq/pkg/yqlib/string_evaluator_test.go b/external/yq/pkg/yqlib/string_evaluator_test.go deleted file mode 100755 index e8e7ddf..0000000 --- a/external/yq/pkg/yqlib/string_evaluator_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package yqlib - -import ( - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -func TestStringEvaluator_Evaluate_Nominal(t *testing.T) { - expected_output := `` + - `yq` + "\n" + - `---` + "\n" + - `jq` + "\n" - expression := ".[].name" - input := `` + - ` - name: yq` + "\n" + - ` description: yq is a portable command-line YAML, JSON and XML processor` + "\n" + - `---` + "\n" + - ` - name: jq` + "\n" + - ` description: Command-line JSON processor` + "\n" - encoder := NewYamlEncoder(2, true, ConfiguredYamlPreferences) - decoder := NewYamlDecoder(ConfiguredYamlPreferences) - - result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder) - if err != nil { - t.Error(err) - } - - test.AssertResult(t, expected_output, result) -} diff --git a/external/yq/pkg/yqlib/utils.go b/external/yq/pkg/yqlib/utils.go deleted file mode 100755 index a1afff2..0000000 --- a/external/yq/pkg/yqlib/utils.go +++ /dev/null @@ -1,63 +0,0 @@ -package yqlib - -import ( - "bufio" - "container/list" - "errors" - "fmt" - "io" - "os" -) - -func readStream(filename string) (io.Reader, error) { - var reader *bufio.Reader - if filename == "-" { - reader = bufio.NewReader(os.Stdin) - } else { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside thar directory. - file, err := os.Open(filename) // #nosec - if err != nil { - return nil, err - } - reader = bufio.NewReader(file) - } - return reader, nil - -} - -func writeString(writer io.Writer, txt string) error { - _, errorWriting := writer.Write([]byte(txt)) - return errorWriting -} - -func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { - err := decoder.Init(reader) - if err != nil { - return nil, err - } - inputList := list.New() - var currentIndex uint - - for { - candidateNode, errorReading := decoder.Decode() - - if errors.Is(errorReading, io.EOF) { - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } - return inputList, nil - } else if errorReading != nil { - return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) - } - candidateNode.Document = currentIndex - candidateNode.Filename = filename - candidateNode.FileIndex = fileIndex - candidateNode.EvaluateTogether = true - - inputList.PushBack(candidateNode) - - currentIndex = currentIndex + 1 - } -} diff --git a/external/yq/pkg/yqlib/write_in_place_handler.go b/external/yq/pkg/yqlib/write_in_place_handler.go deleted file mode 100755 index d3bbb48..0000000 --- a/external/yq/pkg/yqlib/write_in_place_handler.go +++ /dev/null @@ -1,52 +0,0 @@ -package yqlib - -import ( - "os" -) - -type writeInPlaceHandler interface { - CreateTempFile() (*os.File, error) - FinishWriteInPlace(evaluatedSuccessfully bool) error -} - -type writeInPlaceHandlerImpl struct { - inputFilename string - tempFile *os.File -} - -func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler { - - return &writeInPlaceHandlerImpl{inputFile, nil} -} - -func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) { - file, err := createTempFile() - - if err != nil { - return nil, err - } - info, err := os.Stat(w.inputFilename) - if err != nil { - return nil, err - } - err = os.Chmod(file.Name(), info.Mode()) - - if err != nil { - return nil, err - } - log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name()) - w.tempFile = file - return file, err -} - -func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error { - log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) - safelyCloseFile(w.tempFile) - if evaluatedSuccessfully { - log.Debug("Moving temp file to target") - return tryRenameFile(w.tempFile.Name(), w.inputFilename) - } - tryRemoveTempFile(w.tempFile.Name()) - - return nil -} diff --git a/external/yq/pkg/yqlib/xml.go b/external/yq/pkg/yqlib/xml.go deleted file mode 100755 index 09bb872..0000000 --- a/external/yq/pkg/yqlib/xml.go +++ /dev/null @@ -1,29 +0,0 @@ -package yqlib - -type XmlPreferences struct { - AttributePrefix string - ContentName string - StrictMode bool - KeepNamespace bool - UseRawToken bool - ProcInstPrefix string - DirectiveName string - SkipProcInst bool - SkipDirectives bool -} - -func NewDefaultXmlPreferences() XmlPreferences { - return XmlPreferences{ - AttributePrefix: "+@", - ContentName: "+content", - StrictMode: false, - KeepNamespace: true, - UseRawToken: true, - ProcInstPrefix: "+p_", - DirectiveName: "+directive", - SkipProcInst: false, - SkipDirectives: false, - } -} - -var ConfiguredXMLPreferences = NewDefaultXmlPreferences() diff --git a/external/yq/pkg/yqlib/xml_test.go b/external/yq/pkg/yqlib/xml_test.go deleted file mode 100755 index be55325..0000000 --- a/external/yq/pkg/yqlib/xml_test.go +++ /dev/null @@ -1,660 +0,0 @@ -package yqlib - -import ( - "bufio" - "fmt" - "testing" - - "github.com/mikefarah/yq/v4/test" -) - -const inputXMLWithComments = ` - - - - 3 - - - - z - - - - - - -` -const inputXMLWithCommentsWithSubChild = ` - - - - 3 - - - - - - - - - - -` - -const expectedDecodeYamlWithSubChild = `# before cat -cat: - # in cat before - x: "3" # multi - # line comment - # for x - # before y - - y: - # in y before - d: - # in d before - z: - +@sweet: cool - # in d after - # in y after - # in_cat_after -# after cat -` - -const inputXMLWithCommentsWithArray = ` - - - - 3 - - - - - - - - - - - -` - -const expectedDecodeYamlWithArray = `# before cat -cat: - # in cat before - x: "3" # multi - # line comment - # for x - # before y - - y: - # in y before - d: - - # in d before - z: - +@sweet: cool - # in d after - - # in d2 before - z: - +@sweet: cool2 - # in d2 after - # in y after - # in_cat_after -# after cat -` - -const expectedDecodeYamlWithComments = `# before cat -cat: - # in cat before - x: "3" # multi - # line comment - # for x - # before y - - y: - # in y before - # in d before - d: z # in d after - # in y after - # in_cat_after -# after cat -` - -const expectedRoundtripXMLWithComments = ` - 3 - - z - - -` - -const yamlWithComments = `# header comment -# above_cat -cat: # inline_cat - # above_array - array: # inline_array - - val1 # inline_val1 - # above_val2 - - val2 # inline_val2 -# below_cat -` - -const expectedXMLWithComments = ` - val1 - val2 - -` - -const inputXMLWithNamespacedAttr = ` - -` - -const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0" -map: - +@xmlns: some-namespace - +@xmlns:xsi: some-instance - +@xsi:schemaLocation: some-url -` - -const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0" -map: - +@xmlns: some-namespace - +@xmlns:xsi: some-instance - +@xsi:schemaLocation: some-url -` - -const expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version="1.0" -map: - +@xmlns: some-namespace - +@xmlns:xsi: some-instance - +@some-instance:schemaLocation: some-url -` - -const xmlWithCustomDtd = ` - - - -]> - - &writer;©right; -` - -const expectedDtd = ` - - -]> - - &writer;&copyright; - -` - -const expectedSkippedDtd = ` - - &writer;&copyright; - -` - -const xmlWithProcInstAndDirectives = ` - - - - - things - -` - -const yamlWithProcInstAndDirectives = `+p_xml: version="1.0" -+directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" ' -apple: - +p_coolioo: version="1.0" - +directive: 'CATYPE meow purr puss ' - b: things -` - -const expectedXmlWithProcInstAndDirectives = ` - - - things - -` - -var xmlScenarios = []formatScenario{ - { - description: "Parse xml: simple", - subdescription: "Notice how all the values are strings, see the next example on how you can fix that.", - input: "\n\n meow\n 4\n true\n", - expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n", - }, - { - description: "Parse xml: number", - subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.", - input: "\n\n meow\n 4\n true\n", - expression: " (.. | select(tag == \"!!str\")) |= from_yaml", - expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: 4\n cute: true\n", - }, - { - description: "Parse xml: array", - subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.", - input: "\ncat\ngoat", - expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n", - }, - { - description: "Parse xml: attributes", - subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.", - input: "\n\n 7\n", - expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n", - }, - { - description: "Parse xml: attributes with content", - subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.", - input: "\nmeow", - expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n", - }, - { - description: "Parse xml: custom dtd", - subdescription: "DTD entities are processed as directives.", - input: xmlWithCustomDtd, - expected: expectedDtd, - scenarioType: "roundtrip", - }, - { - description: "Roundtrip with name spaced attributes", - skipDoc: true, - input: inputXMLWithNamespacedAttr, - expected: inputXMLWithNamespacedAttr, - scenarioType: "roundtrip", - }, - { - description: "Parse xml: skip custom dtd", - subdescription: "DTDs are directives, skip over directives to skip DTDs.", - input: xmlWithCustomDtd, - expected: expectedSkippedDtd, - scenarioType: "roundtrip-skip-directives", - }, - { - description: "Parse xml: with comments", - subdescription: "A best attempt is made to preserve comments.", - input: inputXMLWithComments, - expected: expectedDecodeYamlWithComments, - scenarioType: "decode", - }, - { - description: "Empty doc", - skipDoc: true, - input: "", - expected: "\n", - scenarioType: "decode", - }, - { - description: "Empty single node", - skipDoc: true, - input: "", - expected: "a:\n", - scenarioType: "decode", - }, - { - description: "Empty close node", - skipDoc: true, - input: "", - expected: "a:\n", - scenarioType: "decode", - }, - { - description: "Nested empty", - skipDoc: true, - input: "", - expected: "a:\n b:\n", - scenarioType: "decode", - }, - { - description: "Parse xml: with comments subchild", - skipDoc: true, - input: inputXMLWithCommentsWithSubChild, - expected: expectedDecodeYamlWithSubChild, - scenarioType: "decode", - }, - { - description: "Parse xml: with comments array", - skipDoc: true, - input: inputXMLWithCommentsWithArray, - expected: expectedDecodeYamlWithArray, - scenarioType: "decode", - }, - { - description: "Parse xml: keep attribute namespace", - subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.KeepNamespace), - skipDoc: false, - input: inputXMLWithNamespacedAttr, - expected: expectedYAMLWithNamespacedAttr, - scenarioType: "decode-keep-ns", - }, - { - description: "Parse xml: keep raw attribute namespace", - skipDoc: true, - input: inputXMLWithNamespacedAttr, - expected: expectedYAMLWithRawNamespacedAttr, - scenarioType: "decode-raw-token", - }, - { - description: "Parse xml: keep raw attribute namespace", - subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.UseRawToken), - skipDoc: false, - input: inputXMLWithNamespacedAttr, - expected: expectedYAMLWithoutRawNamespacedAttr, - scenarioType: "decode-raw-token-off", - }, - { - description: "Encode xml: simple", - input: "cat: purrs", - expected: "purrs\n", - scenarioType: "encode", - }, - { - description: "includes map tags", - skipDoc: true, - input: "purrs\n", - expression: `tag`, - expected: "!!map\n", - scenarioType: "decode", - }, - { - description: "includes array tags", - skipDoc: true, - input: "purrspurrs\n", - expression: `.cat | tag`, - expected: "!!seq\n", - scenarioType: "decode", - }, - { - description: "Encode xml: array", - input: "pets:\n cat:\n - purrs\n - meows", - expected: "\n purrs\n meows\n\n", - scenarioType: "encode", - }, - { - description: "Encode xml: attributes", - subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.", - input: "cat:\n +@name: tiger\n meows: true\n", - expected: "\n true\n\n", - scenarioType: "encode", - }, - { - description: "double prefix", - skipDoc: true, - input: "cat:\n +@+@name: tiger\n meows: true\n", - expected: "\n true\n\n", - scenarioType: "encode", - }, - { - description: "arrays cannot be encoded", - skipDoc: true, - input: "[cat, dog, fish]", - expectedError: "cannot encode !!seq to XML - only maps can be encoded", - scenarioType: "encode-error", - }, - { - description: "arrays cannot be encoded - 2", - skipDoc: true, - input: "[cat, dog]", - expectedError: "cannot encode !!seq to XML - only maps can be encoded", - scenarioType: "encode-error", - }, - { - description: "scalars cannot be encoded", - skipDoc: true, - input: "mike", - expectedError: "cannot encode !!str to XML - only maps can be encoded", - scenarioType: "encode-error", - }, - { - description: "Encode xml: attributes with content", - subdescription: "Fields with the matching xml-content-name is assumed to be content.", - input: "cat:\n +@name: tiger\n +content: cool\n", - expected: "cool\n", - scenarioType: "encode", - }, - { - description: "Encode xml: comments", - subdescription: "A best attempt is made to copy comments to xml.", - input: yamlWithComments, - expected: expectedXMLWithComments, - scenarioType: "encode", - }, - { - description: "Encode: doctype and xml declaration", - subdescription: "Use the special xml names to add/modify proc instructions and directives.", - input: yamlWithProcInstAndDirectives, - expected: expectedXmlWithProcInstAndDirectives, - scenarioType: "encode", - }, - { - description: "Round trip: with comments", - subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.", - input: inputXMLWithComments, - expected: expectedRoundtripXMLWithComments, - scenarioType: "roundtrip", - }, - { - description: "Roundtrip: with doctype and declaration", - subdescription: "yq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.", - input: xmlWithProcInstAndDirectives, - expected: expectedXmlWithProcInstAndDirectives, - scenarioType: "roundtrip", - }, -} - -func testXMLScenario(t *testing.T, s formatScenario) { - switch s.scenarioType { - case "", "decode": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description) - case "encode": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) - case "roundtrip": - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)), s.description) - case "decode-keep-ns": - prefs := NewDefaultXmlPreferences() - prefs.KeepNamespace = true - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "decode-raw-token": - prefs := NewDefaultXmlPreferences() - prefs.UseRawToken = true - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "decode-raw-token-off": - prefs := NewDefaultXmlPreferences() - prefs.UseRawToken = false - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description) - case "roundtrip-skip-directives": - prefs := NewDefaultXmlPreferences() - prefs.SkipDirectives = true - test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)), s.description) - case "encode-error": - result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, NewDefaultXmlPreferences())) - if err == nil { - t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) - } else { - test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) - } - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) { - s := i.(formatScenario) - - if s.skipDoc { - return - } - switch s.scenarioType { - case "", "decode": - documentXMLDecodeScenario(w, s) - case "encode": - documentXMLEncodeScenario(w, s) - case "roundtrip": - documentXMLRoundTripScenario(w, s) - case "decode-keep-ns": - documentXMLDecodeKeepNsScenario(w, s) - case "decode-raw-token-off": - documentXMLDecodeKeepNsRawTokenScenario(w, s) - case "roundtrip-skip-directives": - documentXMLSkipDirectrivesScenario(w, s) - - default: - panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) - } -} - -func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.xml file of:\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - expression := s.expression - if expression == "" { - expression = "." - } - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression)) - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) -} - -func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.xml file of:\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace=false '.' sample.xml\n```\n") - writeOrPanic(w, "will output\n") - prefs := NewDefaultXmlPreferences() - prefs.KeepNamespace = false - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)))) - - prefsWithout := NewDefaultXmlPreferences() - prefs.KeepNamespace = true - writeOrPanic(w, "instead of\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout)))) -} - -func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.xml file of:\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-raw-token=false '.' sample.xml\n```\n") - writeOrPanic(w, "will output\n") - - prefs := NewDefaultXmlPreferences() - prefs.UseRawToken = false - - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)))) - - prefsWithout := NewDefaultXmlPreferences() - prefsWithout.UseRawToken = true - - writeOrPanic(w, "instead of\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(2, prefsWithout)))) -} - -func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.yml file of:\n") - writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -o=xml '.' sample.yml\n```\n") - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) -} - -func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.xml file of:\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n") - writeOrPanic(w, "will output\n") - - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) -} - -func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) { - writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) - - if s.subdescription != "" { - writeOrPanic(w, s.subdescription) - writeOrPanic(w, "\n\n") - } - - writeOrPanic(w, "Given a sample.xml file of:\n") - writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) - - writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-skip-directives '.' sample.xml\n```\n") - writeOrPanic(w, "will output\n") - prefs := NewDefaultXmlPreferences() - prefs.SkipDirectives = true - - writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(2, prefs)))) -} - -func TestXMLScenarios(t *testing.T) { - for _, tt := range xmlScenarios { - testXMLScenario(t, tt) - } - genericScenarios := make([]interface{}, len(xmlScenarios)) - for i, s := range xmlScenarios { - genericScenarios[i] = s - } - documentScenarios(t, "usage", "xml", genericScenarios, documentXMLScenario) -} diff --git a/external/yq/pkg/yqlib/yaml.go b/external/yq/pkg/yqlib/yaml.go deleted file mode 100755 index d7c8863..0000000 --- a/external/yq/pkg/yqlib/yaml.go +++ /dev/null @@ -1,19 +0,0 @@ -package yqlib - -type YamlPreferences struct { - LeadingContentPreProcessing bool - PrintDocSeparators bool - UnwrapScalar bool - EvaluateTogether bool -} - -func NewDefaultYamlPreferences() YamlPreferences { - return YamlPreferences{ - LeadingContentPreProcessing: true, - PrintDocSeparators: true, - UnwrapScalar: true, - EvaluateTogether: false, - } -} - -var ConfiguredYamlPreferences = NewDefaultYamlPreferences() diff --git a/external/yq/release_instructions.txt b/external/yq/release_instructions.txt deleted file mode 100755 index f7cd58b..0000000 --- a/external/yq/release_instructions.txt +++ /dev/null @@ -1,52 +0,0 @@ -- update release notes -- increment version in version.go -- increment version in snapcraft.yaml -- make sure local build passes -- run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch) -- commit version update changes -- tag git with same version number -- commit vX tag - this will trigger github actions -- use github actions to publish docker and make github release -- check github updated yq action in marketplace - - -- snapcraft - - will auto create a candidate, test it works then promote - - !! need to update v4/stable as well as latest - - sudo snap remove yq - sudo snap install --edge yq - - then use the UI (https://snapcraft.io/yq/release) - -- brew - - brew bump-formula-pr --url=https://github.com/mikefarah/yq/archive/2.2.0.tar.gz yq - - if that fails with random ruby errors try: - - clearing out the gems rm -rf .gem/ruby/2.3.0 - - export HOMEBREW_FORCE_VENDOR_RUBY=1 - -- docker - - build and push latest and new version tag - - docker build . -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X - -- debian package (with release script) - - execute the script `./scripts/release-deb.sh` *on the machine having the private - gpg key to sign the generated sources* providing the version to release, the - ppa where deploying the files and the passphrase of the private key if needed: - ``` - ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION - ``` - -- debian package (manually) - - ensure you get all vendor dependencies before packaging - ```go mod vendor``` - - execute - ```dch -i``` - - fill debian/changelog with changes from last version - - build the package sources - ```debuild -i -I -S -sa``` - (signing with gpg key is required in order to put it to ppa) - - put to PPA - ```dput ppa: ../yq__source.changes``` - (current distro repository is ppa:rmescandon/yq. In case that a new version - is released, please contact rmescandon@gmail.com to bump debian package) diff --git a/external/yq/release_notes.txt b/external/yq/release_notes.txt deleted file mode 100755 index 08c05de..0000000 --- a/external/yq/release_notes.txt +++ /dev/null @@ -1,269 +0,0 @@ -4.30.2: - - Actually updated the default xml prefix :facepalm: - -4.30.1: - - XML users note: the default attribute prefix has change to `+@` to avoid naming conflicts! - - Can use expressions in slice #1419 - - Fixed unhandled exception when decoding CSV thanks @washanhanzi - - Added array_to_map operator for #1415 - - Fixed sorting by date #1412 - - Added check to ensure only maps can be encoded to XML #1408 - - Check merge alias is a map #1425 - - Explicity setting unwrap flag works for json output #437, #1409 - - Bumped go version - - - -4.29.2: - - Fixed null pointer exception when parsing CSV with empty field #1404 - -4.29.1: - - Fixed Square brackets removing update #1342 - - Added slice array operator (.[10:15]) #44 - - XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them. - - XML users note that the default attribute prefix will change to `+@` in the 4.30 release to avoid naming conflicts! - - Improved comment handling of decoders (breaking change for yqlib users sorry) - - Fixed load operator bug when loading yaml file with multiple documents - - Bumped Go compiler version - - Bumped dependencies - -4.28.2: - - Fixed Github Actions issues (thanks @mattphelps-8451) - - Fixed bug - can now delete documents #1377 - - Fixed handling of UTF8 encoded CSVs #1373 - - Detect and fail on missing closing brackets #1366 - - yq Github actions now build docker image as part of release - - Bumped dependencies - -4.28.1: - - Added `setpath` and `delpaths` operators, like jq (#1374) - - Added `is_key` operator, to check if a match was a key when recursing - - Added validation when attempting to add sequences to maps (#1341) - -4.27.5: - - Fixed relative merge bug #1333 - -4.27.4: - - Fixed bug in alternative (//) operator, RHS being evaluated when it didn't need to be - - Fixed footer comment issue #1231 - - Github action now runs as root (as recommended by Github Actions doc) - - Updated dependencies - -4.27.3: - - Added new 'c' merge and assign flag that clobbers custom tags - - Bumped go dependency to fix CVE (#1316) - - Updated dependencies - -4.27.2: - - Fixed JSON decoder to maintain object key order. - -4.27.1: - - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON) - - Added 'csv' decoding, array of objects encoding, and round-triping - - New StringEvaluator when using yq as a lib (thanks @leviliangtw) - - Fixed XML decoding issue (#1284) - -4.26.1: - - Switched to new expression parser (#1264) - - Don't clobber anchor when adding nodes (#1269) - - New error operator for custom validation (#1259) - - Added support for --wrapScalar=false in properties encoder (#1241) Thanks @dcarbone - - Fix error on multiple assign (#1257) Thanks @care0717 - - Bumped dependency versions - -4.25.4: - - Fixed panic when using multiply assign on multiple documents #1256 Thanks @care0717 - -4.25.3: - - xml decoder now maintains namespaces by default. Use new flags to disable if required. Thanks @rndmit - - Length and other similar operators no longer return comments (#1231) - - When split expression includes an extension, dont add .yml automatically (#1165) - - Map -r to --unwrapScalar to be more a drop in replacement for jq (#1245) Thanks @SuperSandro2000 - - Fixing usage of quoted numeric keys #1247 - - Bumped dependency versions - - -4.25.2: - - Fixed comments disappearing from end of file (#1217) - - Fixed empty base64 decoding error (#1209) - - JSON output now in colors (#1208) - - Added codeql and fixed minor issues - - Bumped go-yaml library - - Bumped go dependency - -4.25.1: -- Can specify a split expression file #1194 -- Fixed append map bug when key matches value in existing map #1200 -- Nicer error message when trying to use merge anchor tags other than maps #1184 -- Fixed Don't automatically read stdin when the null input flag is used -- Added type as an alias for tag #1195 -- Fixes bug when using write in-place with no expression and multiple files #1193 - -4.24.5: - - Fixed scenarios that dropped the first line if it's a comment (#1181) - - Fixed updating existing empty map resulting in single line styling (#1176) - - Fixed `with` operation bug (#1174) - - Bumped go compiler - -4.24.4: - - Fixed docker release build - - -4.24.3: - - Added from_props - - Re-releasing, 4.24.2 release failed to publish correctly. - - -4.24.2: - - Fixing release pipeline for go1.18 - -4.24.1: - - Added comparison operators! (#94) - - Bumped golang to 1.18 (#1153) - - XML parser no longer runs in strict mode (added new flag to run in strict mode) (#1155) - -4.23.1: - - Can now supply the envsubst operator with parameters (nounset, noempty, failfast). See [envsubst](https://mikefarah.gitbook.io/yq/operators/env-variable-operators) for details (#1137) - - Bumped dependencies - - Fixed '+=' problem with multiple matches #1145 - - Fixed bug with "and", "or" evaluating the RHS when not needed - - Fixed potential panic (thanks @mkatychev) - - Tweaked CLI help (thanks @justin-f-perez) - -4.22.1: - - Added [pick] (https://mikefarah.gitbook.io/yq/operators/pick) operator - - Can load expression from a file '--from-file' (#1120) - - Fixed property auto expansion (#1127) - -4.21.1: - - Added [reverse](https://mikefarah.gitbook.io/yq/operators/reverse) operator - - Added [string case](https://mikefarah.gitbook.io/yq/operators/string-operators) operators - - Added [base64 support](https://mikefarah.gitbook.io/yq/operators/encode-decode) - - Added [line](https://mikefarah.gitbook.io/yq/operators/line) and [column](https://mikefarah.gitbook.io/yq/operators/column) operators - - Bumped dependency versions - -4.20.2: - - Fixed self assignment issue (#1107) - - Fixed bad capture groups with multiple matches (#1114) - - No longer auto-read from STDIN if there are files given (#1115) - - Added missing load_props operator - -4.20.1: - - New [Date Operators](https://mikefarah.gitbook.io/yq/operators/datetime) (now, tz, add and subtract durations from dates) - - Can now decode property files! - - New flag to manually set expression if required - - ZSH completion bug fix (#1108) thanks @whi-tw - - Fixed SEGV error (#1096) - - Fixed Github actions issues (it pipes in /dev/null) for XML - - Fixed bug - handle expressions that match a directory (e.g. ".") - -4.19.1: - - New [eval](https://mikefarah.gitbook.io/yq/operators/eval) _operator_ that allows dynamic expression evaluation (e.g. from a env variable) (#1087) - - Adding new elements to array now automatically applies styling of existing elements (#722) - -4.18.1: - - `eval` is now the _default_ command, you can leave it out #113 - - `-` no longer needs to be specified as STDIN, unless you are also working with multiple files. #113 - - Adding to empty maps / arrays now uses idiomatic yaml styling by default - - Fixed seg fault on bad input #1086 - - New `envsubst` operator! (thanks @sciyoshi) - - Added support for `*=`, relative multiply/merge - - Custom tag types now autocast to there actual types #933 - -4.17.2: - - Fixed manpath issue (thanks @mr-pmillz) - -4.17.1: - - Added XML support (#491) - - New merge flag (n) to only merge new fields (#1038) - - Fixed exit status bug for permission denied error (#1062) - - Fixed using multiple variables with union (,) operator (#1048) - - Bumped some versions of dependencies - -4.16.2: - - Bumped go-lang compiler to fix CVE-2021-44717 (#1037) - - Dependency version bumps via dependabot - - Added extract-checksum.sh to make it easier to validate checksums (#1011) - - Report filename on parsing error (#1030) - -4.16.1: - - Added csv, tsv output formats - - Added map, map_values operators - - Added sort, sort_by operators (#947, #1024) - - Fixed bug in collect - - Fixed permissions issue in Dockerfile (#1014) - - Fixed assignment operator to no longer overwrite anchor (#1029) - -4.15.1: - - Added 'load/strload' operators for dynamically loading content from files - - Added 'key' operator - - Added 'parent' operator - - Smarter MAN page installation script (thanks @coolaj86) - - Dockerfile improvements (thanks @actualben) - - Error handling improvements (thanks @mmorel-35) - -4.14.2: - - Fixed header preprocessing issue (#1000) - - Bumped version dependencies - -4.14.1: - - Added group_by operator - - Added encode/decode operators (toyaml, fromjson etc) (#974) - - Added flatten operator - - Added --split-exp, for splitting results into multiple files (#966) - - Fixed json null array bug (#985) - -4.13.5: - - Performance improvement for deepMatch (thanks @pmatseykanets) - - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961) - - Fixed expression parsing bug #970 - - Rebuild fixes CVE (#964) - - Bumped docker alpine version - - -4.13.4: - - Fixed select bug (#958) - - Improved performance of `explode` (this will also speed up json conversion) - - Improved performance of `merge` (significantly if your merging a small file into a big one) - -4.13.3: - -- Updated go compiler to 1.17 to fix CVE (#944) - -4.13.2: - -- Fixing Docker build timeout issues when attempting to release - -4.13.1: - -- Update to `with` operator, allow for no leading space on the `;`. - -4.13.0: - -BREAKING CHANGE - the `as` variable operator (e.g. `.a as $x`) now makes a _copy_ of the node(s) at the -path rather than a reference. This is in order to make it work more like the `jq` equivalent. - -This means any updates made to that variable do not update the original. - -There's a new operator `ref` that will make a reference (and allow multiple updates to the original path by referencing the variable). -Sorry for any inconvenience caused!. - - -- New `with` operator for making multiple changes to a given path -- New `contains` operator, works like the `jq` equivalent -- Subtract operator now supports subtracting elements from arrays! -- Fixed Swapping values using variables #934 -- Github Action now properly supports multiline output #936, thanks @pjxiao -- Fixed missing closing bracket validation #932 - -4.12.2: -- Fix processing of hex numbers #929 -- Fixed alternative and union operator issues #930 - -4.12.1: - - Merge comment fix #919 - -4.12.0: -- Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info. -- Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919) -- pretty print yaml 1.1 compatibility (https://github.com/mikefarah/yq/issues/914) diff --git a/external/yq/scripts/acceptance.sh b/external/yq/scripts/acceptance.sh deleted file mode 100755 index 7ec801f..0000000 --- a/external/yq/scripts/acceptance.sh +++ /dev/null @@ -1,10 +0,0 @@ -#! /bin/bash -set -e - -for test in acceptance_tests/*.sh; do - echo "--------------------------------------------------------------" - echo "$test" - echo "--------------------------------------------------------------" - (exec "$test"); -done - diff --git a/external/yq/scripts/check.sh b/external/yq/scripts/check.sh deleted file mode 100755 index 069947f..0000000 --- a/external/yq/scripts/check.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -if command -v golangci-lint &> /dev/null -then - golangci-lint run --verbose -else - ./bin/golangci-lint run --verbose -fi - diff --git a/external/yq/scripts/copy-docs.sh b/external/yq/scripts/copy-docs.sh deleted file mode 100755 index 5c27efa..0000000 --- a/external/yq/scripts/copy-docs.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cp how-it-works.md ../yq-gitbook/. -cp pkg/yqlib/doc/operators/*.md ../yq-gitbook/operators/. -cp pkg/yqlib/doc/usage/*.md ../yq-gitbook/usage/. \ No newline at end of file diff --git a/external/yq/scripts/coverage.sh b/external/yq/scripts/coverage.sh deleted file mode 100755 index 88b4b97..0000000 --- a/external/yq/scripts/coverage.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -set -e - -go test -coverprofile=coverage.out -v $(go list ./... | grep -v -E 'examples' | grep -v -E 'test') -go tool cover -html=coverage.out -o coverage.html diff --git a/external/yq/scripts/devtools.sh b/external/yq/scripts/devtools.sh deleted file mode 100755 index d9feeed..0000000 --- a/external/yq/scripts/devtools.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -set -ex -go mod download golang.org/x/tools@latest -curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0 -wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.13.1 diff --git a/external/yq/scripts/extract-checksum.sh b/external/yq/scripts/extract-checksum.sh deleted file mode 100755 index 914dcfa..0000000 --- a/external/yq/scripts/extract-checksum.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash - -# This script works with checksums_hashes_order and checksums to extract the relevant -# sha of the various yq downloads. You can then use your favourite checksum tool to validate. -# must match an entry in checksums_hashes_order. -# -# Usage: ./extract-checksum.sh -# E.g: ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz -# Outputs: -# yq_linux_amd64.tar.gz acebc9d07aa2d0e482969b2c080ee306e8f58efbd6f2d857eefbce6469da1473 -# -# Usage with rhash: -# ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz | rhash -c - -# -# Tip, if you want the checksum first then the filename (e.g. for the md5sum command) -# then you can pipe the output of this script into awk to switch the fields around: -# -# ./extract-checksum.sh MD5 yq_linux_amd64.tar.gz | awk '{ print $2 " " $1}' | md5sum -c - -# -# - -if [ "$1" == "" ]; then - echo "Please specify at a hash algorithm from the checksum_hashes_order" - echo "Usage: $0 " - exit 1 -fi - -if [ "$2" != "" ]; then - # so we dont match x.tar.gz when 'x' is given - file="$2\s" -else - file="" -fi - -if [ ! -f "checksums_hashes_order" ]; then - echo "This script requires checksums_hashes_order to run" - echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate" - exit 1 -fi - -if [ ! -f "checksums" ]; then - echo "This script requires the checksums file to run" - echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate" - exit 1 -fi - - -grepMatch=$(grep -m 1 -n "$1" checksums_hashes_order) -if [ "$grepMatch" == "" ]; then - echo "Could not find hash algorith '$1' in checksums_hashes_order" - exit 1 -fi - -set -e - -lineNumber=$(echo "$grepMatch" | cut -f1 -d:) - -realLineNumber="$(($lineNumber + 1))" - -grep "$file" checksums | sed 's/ /\t/g' | cut -f1,$realLineNumber \ No newline at end of file diff --git a/external/yq/scripts/format.sh b/external/yq/scripts/format.sh deleted file mode 100755 index 2faf7a7..0000000 --- a/external/yq/scripts/format.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -gofmt -w -s . -go mod tidy -go mod vendor diff --git a/external/yq/scripts/generate-man-page-md.sh b/external/yq/scripts/generate-man-page-md.sh deleted file mode 100755 index 34283fc..0000000 --- a/external/yq/scripts/generate-man-page-md.sh +++ /dev/null @@ -1,16 +0,0 @@ -#! /bin/bash -set -e - -# note that this reqires pandoc to be installed. - -cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md -printf "\n# HOW IT WORKS\n" >> man.md -tail -n +2 how-it-works.md >> man.md - -for f in ./pkg/yqlib/doc/operators/*.md; do - cat "$f" >> man.md -done - -for f in ./pkg/yqlib/doc/usage/*.md; do - cat "$f" >> man.md -done diff --git a/external/yq/scripts/generate-man-page.sh b/external/yq/scripts/generate-man-page.sh deleted file mode 100755 index b39e21b..0000000 --- a/external/yq/scripts/generate-man-page.sh +++ /dev/null @@ -1,11 +0,0 @@ -#! /bin/bash -set -e - -# note that this reqires pandoc to be installed. - -pandoc \ - --variable=title:"YQ" \ - --variable=section:"1" \ - --variable=author:"Mike Farah" \ - --variable=header:"${MAN_HEADER}" \ - --standalone --to man man.md -o yq.1 \ No newline at end of file diff --git a/external/yq/scripts/install-man-page.sh b/external/yq/scripts/install-man-page.sh deleted file mode 100755 index 82715ec..0000000 --- a/external/yq/scripts/install-man-page.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -my_path="$(command -v yq)" - -if [ -z "$my_path" ]; then - echo "'yq' wasn't found in your PATH, so we don't know where to put the man pages." - echo "Please update your PATH to include yq, and run this script again." - exit 1 -fi - -# ex: ~/.local/bin/yq => ~/.local/ -my_prefix="$(dirname "$(dirname "$(command -v yq)")")" -mkdir -p "$my_prefix/share/man/man1/" -cp yq.1 "$my_prefix/share/man/man1/" diff --git a/external/yq/scripts/release-deb.sh b/external/yq/scripts/release-deb.sh deleted file mode 100755 index e6c49fd..0000000 --- a/external/yq/scripts/release-deb.sh +++ /dev/null @@ -1,264 +0,0 @@ -#!/bin/bash -eux -# -# Copyright (C) 2021 Roberto Mier Escandón -# -# This script creates a .deb package file with yq valid for ubuntu 20.04 by default -# You can pass - -DOCKER_IMAGE_NAME=yq-deb-builder -DOCKER_IMAGE_TAG=$(git describe --always --tags) -OUTPUT= -GOVERSION="1.17.4" -KEYID= -MAINTAINER= -DO_PUBLISH= -PPA="rmescandon/yq" -VERSION= -DISTRIBUTION= -DO_SIGN= -PASSPHRASE= - -show_help() { - echo " usage: $(basename "$0") VERSION [options...]" - echo "" - echo " positional arguments" - echo " VERSION" - echo "" - echo " optional arguments:" - echo " -h, --help Shows this help" - echo " -d, --distribution DISTRO The distribution to use for the changelog generation. If not provided, last changelog entry" - echo " distribution is considered" - echo " --goversion VERSION The version of golang to use. Default to $GOVERSION" - echo " -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this" - echo " paramater, the generated sources are not signed" - echo " -s, --sign Sign the package sources with a gpg key of the maintainer" - echo " -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is" - echo " the considered format" - echo " -o DIR, --output DIR The path where leaving the generated debian package. Default to a temporary folder if not set" - echo " -p The resultant file is being published to ppa" - echo " --ppa PPA Push resultant files to indicated ppa. This option should be given along with a signing key." - echo " Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'" - echo " --passphrase PASSPHRASE Passphrase to decrypt the signage key" - exit 1 -} -# read input args -while [ $# -ne 0 ]; do - case $1 in - -h|--help) - show_help - ;; - -d|--distribution) - shift - DISTRIBUTION="$1" - ;; - --goversion) - shift - GOVERSION="$1" - ;; - -k|--sign-key) - shift - DO_SIGN='y' - KEYID="$1" - ;; - -s|--sign) - DO_SIGN='y' - ;; - -m|--maintainer) - shift - MAINTAINER="$1" - ;; - -o|--output) - shift - OUTPUT="$1" - ;; - -p) - DO_PUBLISH="y" - ;; - --ppa) - shift - DO_PUBLISH="y" - PPA="$1" - ;; - --passphrase) - shift - PASSPHRASE="$1" - ;; - *) - if [ -z "$VERSION" ]; then - VERSION="$1" - else - show_help - fi - esac - shift -done - -[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help) - -if [ -n "$OUTPUT" ]; then - OUTPUT="$(realpath "$OUTPUT")" - mkdir -p "$OUTPUT" -else - # Temporary folder where leaving the built deb package in case that output folder is not provided - OUTPUT="$(mktemp -d)" -fi - -# Define the folders with the source project and the build artifacts and files -srcdir="$(realpath "$(dirname "$0")"/..)" -blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")" -# clean on exit -cleanup() { - rm -f "${blddir}/build.sh" || true - rm -f "${blddir}/Dockerfile" || true - rm -f "${blddir}/dput.cf" || true - docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true -} -trap cleanup EXIT INT - -# configure the dput config in case publishing is requested -lp_id="$(echo "$PPA" | cut -d'/' -f1)" -ppa_id="$(echo "$PPA" | cut -d'/' -f2)" -cat << EOF > ${blddir}/dput.cf -[ppa] -fqdn = ppa.launchpad.net -method = ftp -incoming = ~${lp_id}/ubuntu/${ppa_id} -login = anonymous -EOF - -# create the main script -cat << EOF > ${blddir}/build.sh -#!/bin/bash -set -e -o pipefail - -PATH=$PATH:/usr/local/go/bin -export GPG_TTY=$(tty) - -go mod vendor - -### bump debian/changelog - -# maintainer -export DEBEMAIL="$MAINTAINER" -if [ -z "$MAINTAINER" ]; then - export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)" -fi - -# prepend a 'v' char to complete the tag name from where calculating the changelog -SINCE="v\$(dpkg-parsechangelog -S Version)" - -# distribution -DISTRIBUTION="$DISTRIBUTION" -if [ -z "$DISTRIBUTION" ]; then - DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)" -fi - -# generate changelog -gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION" - -# using -d to prevent failing when searching for golang dep on control file -params=("-d" "-S") - -# add the -sa option for signing along with the key to use when provided key id -if [ -n "$DO_SIGN" ]; then - params+=("-sa") - - # read from gpg the key id associated with the maintainer if not provided explicitly - if [ -z "$KEYID" ]; then - KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)" - else - KEYID="$KEYID" - fi - - params+=("--sign-key="\$KEYID"") - - if [ -n "$PASSPHRASE" ]; then - gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000 - KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)" - /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP" - fi - -else - params+=("-us" "-uc") -fi - -debuild \${params[@]} -mv ../yq_* /home/yq/output - -echo "" -echo -e "\tfind resulting package at: "$OUTPUT"" - -# publish to ppa whether given -if [ -n "$DO_PUBLISH" ]; then - dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes -fi -EOF -chmod +x "${blddir}"/build.sh - -# build the docker image with all dependencies -cat << EOF > ${blddir}/Dockerfile -FROM bitnami/minideb:bullseye as base -ENV LANG C.UTF-8 -ENV LC_ALL C.UTF-8 -ENV DEBIAN_FRONTEND noninteractive -ENV GO111MODULE on -ENV GOMODCACHE /home/yq/go - -RUN set -e \ - && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \ - && apt-get -qq update - -# install golang on its $GOVERSION -FROM base as golang -RUN apt-get -qq -y --no-install-recommends install \ - ca-certificates \ - wget -RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4 -RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz" - -FROM base -RUN apt-get -qq -y --no-install-recommends install \ - build-essential \ - debhelper \ - devscripts \ - dput \ - fakeroot \ - git-buildpackage \ - gpg-agent \ - libdistro-info-perl \ - pandoc \ - rsync \ - sensible-utils && \ - apt-get clean && \ - rm -rf /tmp/* /var/tmp/* - -COPY --from=golang /usr/local/go /usr/local/go - -# build debian package as yq user -RUN useradd -ms /bin/bash yq && \ - mkdir /home/yq/src && chown -R yq: /home/yq/src && \ - mkdir /home/yq/output && chown -R yq: /home/yq/output - -ADD ./build/dput.cf /etc/dput.cf -ADD ./build/build.sh /usr/bin/build.sh -RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh - -USER yq - -WORKDIR /home/yq/src -VOLUME ["/home/yq/src"] - -# dir where output packages are finally left -VOLUME ["/home/yq/output"] - -CMD ["/usr/bin/build.sh"] -EOF - -DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" . - -docker run --rm -i \ - -v "${srcdir}":/home/yq/src:delegated \ - -v "${OUTPUT}":/home/yq/output \ - -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \ - -u "$(id -u)" \ - "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" diff --git a/external/yq/scripts/secure.sh b/external/yq/scripts/secure.sh deleted file mode 100755 index 956961d..0000000 --- a/external/yq/scripts/secure.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail - -if command -v gosec &> /dev/null -then - gosec "${PWD}" ./... -else - ./bin/gosec "${PWD}" ./... -fi \ No newline at end of file diff --git a/external/yq/scripts/setup.sh b/external/yq/scripts/setup.sh deleted file mode 100755 index 1089fde..0000000 --- a/external/yq/scripts/setup.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -set -eu - -find_mgr() { - if hash minishift 2>/dev/null; then - echo "minishift" - else - if hash docker-machine 2>/dev/null; then - echo "docker-machine" - fi - fi -} - -get_vm_name() { - case "$1" in - minishift) - echo "minishift" - ;; - docker-machine) - echo "${DOCKER_MACHINE_NAME}" - ;; - *) - ;; - esac -} - -is_vm_running() { - local vm=$1 - declare -a running=($(VBoxManage list runningvms | awk '{ print $1 }')) - local result='false' - - for rvm in "${running[@]}"; do - if [[ "${rvm}" == *"${vm}"* ]]; then - result='true' - fi - done - echo "$result" -} - -if hash cygpath 2>/dev/null; then - PROJECT_DIR=$(cygpath -w -a "$(pwd)") -else - PROJECT_DIR=$(pwd) -fi - -VM_MGR=$(find_mgr) -if [[ -z $VM_MGR ]]; then - echo "ERROR: No VM Manager found; expected one of ['minishift', 'docker-machine']" - exit 1 -fi - -VM_NAME=$(get_vm_name "$VM_MGR") -if [[ -z $VM_NAME ]]; then - echo "ERROR: No VM found; try running 'eval $(docker-machine env)'" - exit 1 -fi - -if ! hash VBoxManage 2>/dev/null; then - echo "VirtualBox executable 'VBoxManage' not found in path" - exit 1 -fi - -avail=$(is_vm_running "$VM_NAME") -if [[ "$avail" == *"true"* ]]; then - res=$(VBoxManage sharedfolder add "${VM_NAME}" --name "${PROJECT}" --hostpath "${PROJECT_DIR}" --transient 2>&1) - if [[ -z $res || $res == *"already exists"* ]]; then - # no need to show that it already exists - : - else - echo "$res" - exit 1 - fi - echo "VM: [${VM_NAME}] -- Added Sharedfolder [${PROJECT}] @Path [${PROJECT_DIR}]" -else - echo "$VM_NAME is not currently running; please start your VM and try again." - exit 1 -fi - -SSH_CMD="sudo mkdir -p /${PROJECT} ; sudo mount -t vboxsf ${PROJECT} /${PROJECT}" -case "${VM_MGR}" in - minishift) - minishift ssh "${SSH_CMD}" - echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]" - ;; - docker-machine) - docker-machine ssh "${VM_NAME}" "${SSH_CMD}" - echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]" - ;; - *) - ;; -esac diff --git a/external/yq/scripts/shunit2 b/external/yq/scripts/shunit2 deleted file mode 100755 index 6239683..0000000 --- a/external/yq/scripts/shunit2 +++ /dev/null @@ -1,1343 +0,0 @@ -#! /bin/sh -# vim:et:ft=sh:sts=2:sw=2 -# -# Copyright 2008-2020 Kate Ward. All Rights Reserved. -# Released under the Apache 2.0 license. -# http://www.apache.org/licenses/LICENSE-2.0 -# -# shUnit2 -- Unit testing framework for Unix shell scripts. -# https://github.com/kward/shunit2 -# -# Author: kate.ward@forestent.com (Kate Ward) -# -# shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is -# based on the popular JUnit unit testing framework for Java. -# -# $() are not fully portable (POSIX != portable). -# shellcheck disable=SC2006 -# expr may be antiquated, but it is the only solution in some cases. -# shellcheck disable=SC2003 - -# Return if shunit2 already loaded. -command [ -n "${SHUNIT_VERSION:-}" ] && exit 0 -SHUNIT_VERSION='2.1.8' - -# Return values that scripts can use. -SHUNIT_TRUE=0 -SHUNIT_FALSE=1 -SHUNIT_ERROR=2 - -# Logging functions. -_shunit_warn() { - ${__SHUNIT_CMD_ECHO_ESC} \ - "${__shunit_ansi_yellow}shunit2:WARN${__shunit_ansi_none} $*" >&2 -} -_shunit_error() { - ${__SHUNIT_CMD_ECHO_ESC} \ - "${__shunit_ansi_red}shunit2:ERROR${__shunit_ansi_none} $*" >&2 -} -_shunit_fatal() { - ${__SHUNIT_CMD_ECHO_ESC} \ - "${__shunit_ansi_red}shunit2:FATAL${__shunit_ansi_none} $*" >&2 - exit ${SHUNIT_ERROR} -} - -# Determine some reasonable command defaults. -__SHUNIT_CMD_ECHO_ESC='echo -e' -# shellcheck disable=SC2039 -command [ "`echo -e test`" = '-e test' ] && __SHUNIT_CMD_ECHO_ESC='echo' - -__SHUNIT_UNAME_S=`uname -s` -case "${__SHUNIT_UNAME_S}" in - BSD) __SHUNIT_CMD_EXPR='gexpr' ;; - *) __SHUNIT_CMD_EXPR='expr' ;; -esac -__SHUNIT_CMD_TPUT='tput' - -# Commands a user can override if needed. -SHUNIT_CMD_EXPR=${SHUNIT_CMD_EXPR:-${__SHUNIT_CMD_EXPR}} -SHUNIT_CMD_TPUT=${SHUNIT_CMD_TPUT:-${__SHUNIT_CMD_TPUT}} - -# Enable color output. Options are 'never', 'always', or 'auto'. -SHUNIT_COLOR=${SHUNIT_COLOR:-auto} - -# Specific shell checks. -if command [ -n "${ZSH_VERSION:-}" ]; then - setopt |grep "^shwordsplit$" >/dev/null - if command [ $? -ne ${SHUNIT_TRUE} ]; then - _shunit_fatal 'zsh shwordsplit option is required for proper operation' - fi - if command [ -z "${SHUNIT_PARENT:-}" ]; then - _shunit_fatal "zsh does not pass \$0 through properly. please declare \ -\"SHUNIT_PARENT=\$0\" before calling shUnit2" - fi -fi - -# -# Constants -# - -__SHUNIT_MODE_SOURCED='sourced' -__SHUNIT_MODE_STANDALONE='standalone' -__SHUNIT_PARENT=${SHUNIT_PARENT:-$0} - -# User provided test prefix to display in front of the name of the test being -# executed. Define by setting the SHUNIT_TEST_PREFIX variable. -__SHUNIT_TEST_PREFIX=${SHUNIT_TEST_PREFIX:-} - -# ANSI colors. -__SHUNIT_ANSI_NONE='\033[0m' -__SHUNIT_ANSI_RED='\033[1;31m' -__SHUNIT_ANSI_GREEN='\033[1;32m' -__SHUNIT_ANSI_YELLOW='\033[1;33m' -__SHUNIT_ANSI_CYAN='\033[1;36m' - -# Set the constants readonly. -__shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` -echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ - __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` -for __shunit_const in ${__shunit_constants}; do - if command [ -z "${ZSH_VERSION:-}" ]; then - readonly "${__shunit_const}" - else - case ${ZSH_VERSION} in - [123].*) readonly "${__shunit_const}" ;; - *) readonly -g "${__shunit_const}" # Declare readonly constants globally. - esac - fi -done -unset __shunit_const __shunit_constants - -# -# Internal variables. -# - -# Variables. -__shunit_lineno='' # Line number of executed test. -__shunit_mode=${__SHUNIT_MODE_SOURCED} # Operating mode. -__shunit_reportGenerated=${SHUNIT_FALSE} # Is report generated. -__shunit_script='' # Filename of unittest script (standalone mode). -__shunit_skip=${SHUNIT_FALSE} # Is skipping enabled. -__shunit_suite='' # Suite of tests to execute. -__shunit_clean=${SHUNIT_FALSE} # _shunit_cleanup() was already called. - -# ANSI colors (populated by _shunit_configureColor()). -__shunit_ansi_none='' -__shunit_ansi_red='' -__shunit_ansi_green='' -__shunit_ansi_yellow='' -__shunit_ansi_cyan='' - -# Counts of tests. -__shunit_testSuccess=${SHUNIT_TRUE} -__shunit_testsTotal=0 -__shunit_testsPassed=0 -__shunit_testsFailed=0 - -# Counts of asserts. -__shunit_assertsTotal=0 -__shunit_assertsPassed=0 -__shunit_assertsFailed=0 -__shunit_assertsSkipped=0 - -# -# Macros. -# - -# shellcheck disable=SC2016,SC2089 -_SHUNIT_LINENO_='eval __shunit_lineno=""; if command [ "${1:-}" = "--lineno" ]; then command [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' - -#----------------------------------------------------------------------------- -# Assertion functions. -# - -# Assert that two values are equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertEquals() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertEquals() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if command [ "${shunit_expected_}" = "${shunit_actual_}" ]; then - _shunit_assertPass - else - failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' - -# Assert that two values are not equal to one another. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotEquals() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotEquals() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_return=${SHUNIT_TRUE} - if command [ "${shunit_expected_}" != "${shunit_actual_}" ]; then - _shunit_assertPass - else - failSame "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' - -# Assert that a container contains a content. -# -# Args: -# message: string: failure message [optional] -# container: string: container to analyze -# content: string: content to find -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertContains() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertContains() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_container_=$1 - shunit_content_=$2 - - shunit_return=${SHUNIT_TRUE} - if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then - _shunit_assertPass - else - failNotFound "${shunit_message_}" "${shunit_content_}" - shunit_return=${SHUNIT_FALSE} - fi - - unset shunit_message_ shunit_container_ shunit_content_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_CONTAINS_='eval assertContains --lineno "${LINENO:-}"' - -# Assert that a container does not contain a content. -# -# Args: -# message: string: failure message [optional] -# container: string: container to analyze -# content: string: content to look for -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotContains() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotContains() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_container_=$1 - shunit_content_=$2 - - shunit_return=${SHUNIT_TRUE} - if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then - failFound "${shunit_message_}" "${shunit_content_}" - shunit_return=${SHUNIT_FALSE} - else - _shunit_assertPass - fi - - unset shunit_message_ shunit_container_ shunit_content_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_NOT_CONTAINS_='eval assertNotContains --lineno "${LINENO:-}"' - -# Assert that a value is null (i.e. an empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNull() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertNull() requires one or two arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertTrue "${shunit_message_}" "[ -z '$1' ]" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' - -# Assert that a value is not null (i.e. a non-empty string) -# -# Args: -# message: string: failure message [optional] -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotNull() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null - _shunit_error "assertNotNull() requires one or two arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` - test -n "${shunit_actual_}" - assertTrue "${shunit_message_}" $? - shunit_return=$? - - unset shunit_actual_ shunit_message_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' - -# Assert that two values are the same (i.e. equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertSame() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertSame() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - assertEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' - -# Assert that two values are not the same (i.e. not equal to one another). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertNotSame() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "assertNotSame() requires two or three arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_:-}$1" - shift - fi - assertNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is true. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertTrue 0 -# assertTrue "[ 34 -gt 23 ]" -# The following test will fail with a message: -# assertTrue 123 -# assertTrue "test failed" "[ -r '/non/existent/file' ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertTrue() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertTrue() takes one or two arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # See if condition is an integer, i.e. a return value. - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if command [ -z "${shunit_condition_}" ]; then - # Null condition. - shunit_return=${SHUNIT_FALSE} - elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # Possible return value. Treating 0 as true, and non-zero as false. - command [ "${shunit_condition_}" -ne 0 ] && shunit_return=${SHUNIT_FALSE} - else - # Hopefully... a condition. - ( eval "${shunit_condition_}" ) >/dev/null 2>&1 - command [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # Record the test. - if command [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' - -# Assert that a value or shell test condition is false. -# -# In shell, a value of 0 is true and a non-zero value is false. Any integer -# value passed can thereby be tested. -# -# Shell supports much more complicated tests though, and a means to support -# them was needed. As such, this function tests that conditions are true or -# false through evaluation rather than just looking for a true or false. -# -# The following test will succeed: -# assertFalse 1 -# assertFalse "[ 'apples' = 'oranges' ]" -# The following test will fail with a message: -# assertFalse 0 -# assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" -# -# Args: -# message: string: failure message [optional] -# condition: string: integer value or shell conditional statement -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -assertFalse() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "assertFalse() requires one or two arguments; $# given" - _shunit_assertFail - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_condition_=$1 - - # See if condition is an integer, i.e. a return value. - shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` - shunit_return=${SHUNIT_TRUE} - if command [ -z "${shunit_condition_}" ]; then - # Null condition. - shunit_return=${SHUNIT_FALSE} - elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] - then - # Possible return value. Treating 0 as true, and non-zero as false. - command [ "${shunit_condition_}" -eq 0 ] && shunit_return=${SHUNIT_FALSE} - else - # Hopefully... a condition. - ( eval "${shunit_condition_}" ) >/dev/null 2>&1 - command [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} - fi - - # Record the test. - if command [ "${shunit_return}" -eq "${SHUNIT_TRUE}" ]; then - _shunit_assertPass - else - _shunit_assertFail "${shunit_message_}" - fi - - unset shunit_message_ shunit_condition_ shunit_match_ - return "${shunit_return}" -} -# shellcheck disable=SC2016,SC2034 -_ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# Failure functions. -# - -# Records a test failure. -# -# Args: -# message: string: failure message [optional] -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -fail() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -gt 1 ]; then - _shunit_error "fail() requires zero or one arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 1 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - _shunit_assertFail "${shunit_message_}" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_='eval fail --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotEquals() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotEquals() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_expected_=$1 - shunit_actual_=$2 - - shunit_message_=${shunit_message_%% } - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" - - unset shunit_message_ shunit_expected_ shunit_actual_ - return ${SHUNIT_FALSE} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' - -# Records a test failure, stating a value was found. -# -# Args: -# message: string: failure message [optional] -# content: string: found value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failFound() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "failFound() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - shunit_message_=${shunit_message_%% } - _shunit_assertFail "${shunit_message_:+${shunit_message_} }Found" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_FOUND_='eval failFound --lineno "${LINENO:-}"' - -# Records a test failure, stating a content was not found. -# -# Args: -# message: string: failure message [optional] -# content: string: content not found -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotFound() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 1 -o $# -gt 2 ]; then - _shunit_error "failNotFound() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 2 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - shunit_content_=$1 - - shunit_message_=${shunit_message_%% } - _shunit_assertFail "${shunit_message_:+${shunit_message_} }Not found:<${shunit_content_}>" - - unset shunit_message_ shunit_content_ - return ${SHUNIT_FALSE} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_NOT_FOUND_='eval failNotFound --lineno "${LINENO:-}"' - -# Records a test failure, stating two values should have been the same. -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failSame() -{ - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failSame() requires two or three arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - - shunit_message_=${shunit_message_%% } - _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" - - unset shunit_message_ - return ${SHUNIT_FALSE} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' - -# Records a test failure, stating two values were not equal. -# -# This is functionally equivalent to calling failNotEquals(). -# -# Args: -# message: string: failure message [optional] -# expected: string: expected value -# actual: string: actual value -# Returns: -# integer: success (TRUE/FALSE/ERROR constant) -failNotSame() { - # shellcheck disable=SC2090 - ${_SHUNIT_LINENO_} - if command [ $# -lt 2 -o $# -gt 3 ]; then - _shunit_error "failNotSame() requires one or two arguments; $# given" - return ${SHUNIT_ERROR} - fi - _shunit_shouldSkip && return ${SHUNIT_TRUE} - - shunit_message_=${__shunit_lineno} - if command [ $# -eq 3 ]; then - shunit_message_="${shunit_message_}$1" - shift - fi - failNotEquals "${shunit_message_}" "$1" "$2" - shunit_return=$? - - unset shunit_message_ - return ${shunit_return} -} -# shellcheck disable=SC2016,SC2034 -_FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' - -#----------------------------------------------------------------------------- -# Skipping functions. -# - -# Force remaining assert and fail functions to be "skipped". -# -# This function forces the remaining assert and fail functions to be "skipped", -# i.e. they will have no effect. Each function skipped will be recorded so that -# the total of asserts and fails will not be altered. -# -# Args: -# None -startSkipping() { __shunit_skip=${SHUNIT_TRUE}; } - -# Resume the normal recording behavior of assert and fail calls. -# -# Args: -# None -endSkipping() { __shunit_skip=${SHUNIT_FALSE}; } - -# Returns the state of assert and fail call skipping. -# -# Args: -# None -# Returns: -# boolean: (TRUE/FALSE constant) -isSkipping() { return ${__shunit_skip}; } - -#----------------------------------------------------------------------------- -# Suite functions. -# - -# Stub. This function should contains all unit test calls to be made. -# -# DEPRECATED (as of 2.1.0) -# -# This function can be optionally overridden by the user in their test suite. -# -# If this function exists, it will be called when shunit2 is sourced. If it -# does not exist, shunit2 will search the parent script for all functions -# beginning with the word 'test', and they will be added dynamically to the -# test suite. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Adds a function name to the list of tests schedule for execution. -# -# This function should only be called from within the suite() function. -# -# Args: -# function: string: name of a function to add to current unit test suite -suite_addTest() { - shunit_func_=${1:-} - - __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" - __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` - - unset shunit_func_ -} - -# Stub. This function will be called once before any tests are run. -# -# Common one-time environment preparation tasks shared by all tests can be -# defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called once after all tests are finished. -# -# Common one-time environment cleanup tasks shared by all tests can be defined -# here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Stub. This function will be called before each test is run. -# -# Common environment preparation tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#setUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -# Note: see _shunit_mktempFunc() for actual implementation -# Stub. This function will be called after each test is run. -# -# Common environment cleanup tasks shared by all tests can be defined here. -# -# This function should be overridden by the user in their unit test suite. -# Note: see _shunit_mktempFunc() for actual implementation -# -# Args: -# None -#tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION - -#------------------------------------------------------------------------------ -# Internal shUnit2 functions. -# - -# Create a temporary directory to store various run-time files in. -# -# This function is a cross-platform temporary directory creation tool. Not all -# OSes have the `mktemp` function, so one is included here. -# -# Args: -# None -# Outputs: -# string: the temporary directory that was created -_shunit_mktempDir() { - # Try the standard `mktemp` function. - ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return - - # The standard `mktemp` didn't work. Use our own. - # shellcheck disable=SC2039 - if command [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then - _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" -#! /bin/sh -exit ${SHUNIT_TRUE} -EOF - command chmod +x "${_shunit_file_}" - done - - unset _shunit_file_ -} - -# Final cleanup function to leave things as we found them. -# -# Besides removing the temporary directory, this function is in charge of the -# final exit code of the unit test. The exit code is based on how the script -# was ended (e.g. normal exit, or via Ctrl-C). -# -# Args: -# name: string: name of the trap called (specified when trap defined) -_shunit_cleanup() { - _shunit_name_=$1 - - case "${_shunit_name_}" in - EXIT) ;; - INT) _shunit_signal_=130 ;; # 2+128 - TERM) _shunit_signal_=143 ;; # 15+128 - *) - _shunit_error "unrecognized trap value (${_shunit_name_})" - _shunit_signal_=0 - ;; - esac - if command [ "${_shunit_name_}" != 'EXIT' ]; then - _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" - fi - - # Do our work. - if command [ ${__shunit_clean} -eq ${SHUNIT_FALSE} ]; then - # Ensure tear downs are only called once. - __shunit_clean=${SHUNIT_TRUE} - - tearDown - command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_warn "tearDown() returned non-zero return code." - oneTimeTearDown - command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_warn "oneTimeTearDown() returned non-zero return code." - - command rm -fr "${__shunit_tmpDir}" - fi - - if command [ "${_shunit_name_}" != 'EXIT' ]; then - # Handle all non-EXIT signals. - trap - 0 # Disable EXIT trap. - exit ${_shunit_signal_} - elif command [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ]; then - _shunit_assertFail 'unknown failure encountered running a test' - _shunit_generateReport - exit ${SHUNIT_ERROR} - fi - - unset _shunit_name_ _shunit_signal_ -} - -# configureColor based on user color preference. -# -# Args: -# color: string: color mode (one of `always`, `auto`, or `none`). -_shunit_configureColor() { - _shunit_color_=${SHUNIT_FALSE} # By default, no color. - case $1 in - 'always') _shunit_color_=${SHUNIT_TRUE} ;; - 'auto') - command [ "`_shunit_colors`" -ge 8 ] && _shunit_color_=${SHUNIT_TRUE} - ;; - 'none') ;; - *) _shunit_fatal "unrecognized color option '$1'" ;; - esac - - case ${_shunit_color_} in - ${SHUNIT_TRUE}) - __shunit_ansi_none=${__SHUNIT_ANSI_NONE} - __shunit_ansi_red=${__SHUNIT_ANSI_RED} - __shunit_ansi_green=${__SHUNIT_ANSI_GREEN} - __shunit_ansi_yellow=${__SHUNIT_ANSI_YELLOW} - __shunit_ansi_cyan=${__SHUNIT_ANSI_CYAN} - ;; - ${SHUNIT_FALSE}) - __shunit_ansi_none='' - __shunit_ansi_red='' - __shunit_ansi_green='' - __shunit_ansi_yellow='' - __shunit_ansi_cyan='' - ;; - esac - - unset _shunit_color_ _shunit_tput_ -} - -# colors returns the number of supported colors for the TERM. -_shunit_colors() { - _shunit_tput_=`${SHUNIT_CMD_TPUT} colors 2>/dev/null` - if command [ $? -eq 0 ]; then - echo "${_shunit_tput_}" - else - echo 16 - fi - unset _shunit_tput_ -} - -# The actual running of the tests happens here. -# -# Args: -# None -_shunit_execSuite() { - for _shunit_test_ in ${__shunit_suite}; do - __shunit_testSuccess=${SHUNIT_TRUE} - - # Disable skipping. - endSkipping - - # Execute the per-test setup function. - setUp - command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_fatal "setup() returned non-zero return code." - - # Execute the test. - echo "${__SHUNIT_TEST_PREFIX}${_shunit_test_}" - eval "${_shunit_test_}" - if command [ $? -ne ${SHUNIT_TRUE} ]; then - _shunit_error "${_shunit_test_}() returned non-zero return code." - __shunit_testSuccess=${SHUNIT_ERROR} - _shunit_incFailedCount - fi - - # Execute the per-test tear-down function. - tearDown - command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_fatal "tearDown() returned non-zero return code." - - # Update stats. - if command [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then - __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` - else - __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` - fi - done - - unset _shunit_test_ -} - -# Generates the user friendly report with appropriate OK/FAILED message. -# -# Args: -# None -# Output: -# string: the report of successful and failed tests, as well as totals. -_shunit_generateReport() { - command [ "${__shunit_reportGenerated}" -eq ${SHUNIT_TRUE} ] && return - - _shunit_ok_=${SHUNIT_TRUE} - - # If no exit code was provided, determine an appropriate one. - command [ "${__shunit_testsFailed}" -gt 0 \ - -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ - && _shunit_ok_=${SHUNIT_FALSE} - - echo - _shunit_msg_="Ran ${__shunit_ansi_cyan}${__shunit_testsTotal}${__shunit_ansi_none}" - if command [ "${__shunit_testsTotal}" -eq 1 ]; then - ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} test." - else - ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} tests." - fi - - if command [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then - _shunit_msg_="${__shunit_ansi_green}OK${__shunit_ansi_none}" - command [ "${__shunit_assertsSkipped}" -gt 0 ] \ - && _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none})" - else - _shunit_msg_="${__shunit_ansi_red}FAILED${__shunit_ansi_none}" - _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_red}failures=${__shunit_assertsFailed}${__shunit_ansi_none}" - command [ "${__shunit_assertsSkipped}" -gt 0 ] \ - && _shunit_msg_="${_shunit_msg_},${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none}" - _shunit_msg_="${_shunit_msg_})" - fi - - echo - ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_}" - __shunit_reportGenerated=${SHUNIT_TRUE} - - unset _shunit_msg_ _shunit_ok_ -} - -# Test for whether a function should be skipped. -# -# Args: -# None -# Returns: -# boolean: whether the test should be skipped (TRUE/FALSE constant) -_shunit_shouldSkip() { - command [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} - _shunit_assertSkip -} - -# Records a successful test. -# -# Args: -# None -_shunit_assertPass() { - __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` - __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` -} - -# Records a test failure. -# -# Args: -# message: string: failure message to provide user -_shunit_assertFail() { - __shunit_testSuccess=${SHUNIT_FALSE} - _shunit_incFailedCount - - \[ $# -gt 0 ] && ${__SHUNIT_CMD_ECHO_ESC} \ - "${__shunit_ansi_red}ASSERT:${__shunit_ansi_none}$*" -} - -# Increment the count of failed asserts. -# -# Args: -# none -_shunit_incFailedCount() { - __shunit_assertsFailed=`expr "${__shunit_assertsFailed}" + 1` - __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` -} - - -# Records a skipped test. -# -# Args: -# None -_shunit_assertSkip() { - __shunit_assertsSkipped=`expr "${__shunit_assertsSkipped}" + 1` - __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` -} - -# Prepare a script filename for sourcing. -# -# Args: -# script: string: path to a script to source -# Returns: -# string: filename prefixed with ./ (if necessary) -_shunit_prepForSourcing() { - _shunit_script_=$1 - case "${_shunit_script_}" in - /*|./*) echo "${_shunit_script_}" ;; - *) echo "./${_shunit_script_}" ;; - esac - unset _shunit_script_ -} - -# Escape a character in a string. -# -# Args: -# c: string: unescaped character -# s: string: to escape character in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharInStr() { - command [ -n "$2" ] || return # No point in doing work on an empty string. - - # Note: using shorter variable names to prevent conflicts with - # _shunit_escapeCharactersInString(). - _shunit_c_=$1 - _shunit_s_=$2 - - # Escape the character. - # shellcheck disable=SC1003,SC2086 - echo ''${_shunit_s_}'' |command sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' - - unset _shunit_c_ _shunit_s_ -} - -# Escape a character in a string. -# -# Args: -# str: string: to escape characters in -# Returns: -# string: with escaped character(s) -_shunit_escapeCharactersInString() { - command [ -n "$1" ] || return # No point in doing work on an empty string. - - _shunit_str_=$1 - - # Note: using longer variable names to prevent conflicts with - # _shunit_escapeCharInStr(). - for _shunit_char_ in '"' '$' "'" '`'; do - _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` - done - - echo "${_shunit_str_}" - unset _shunit_char_ _shunit_str_ -} - -# Extract list of functions to run tests against. -# -# Args: -# script: string: name of script to extract functions from -# Returns: -# string: of function names -_shunit_extractTestFunctions() { - _shunit_script_=$1 - - # Extract the lines with test function names, strip of anything besides the - # function name, and output everything on a single line. - _shunit_regex_='^\s*((function test[A-Za-z0-9_-]*)|(test[A-Za-z0-9_-]* *\(\)))' - # shellcheck disable=SC2196 - egrep "${_shunit_regex_}" "${_shunit_script_}" \ - |command sed 's/^[^A-Za-z0-9_-]*//;s/^function //;s/\([A-Za-z0-9_-]*\).*/\1/g' \ - |xargs - - unset _shunit_regex_ _shunit_script_ -} - -#------------------------------------------------------------------------------ -# Main. -# - -# Determine the operating mode. -if command [ $# -eq 0 -o "${1:-}" = '--' ]; then - __shunit_script=${__SHUNIT_PARENT} - __shunit_mode=${__SHUNIT_MODE_SOURCED} -else - __shunit_script=$1 - command [ -r "${__shunit_script}" ] || \ - _shunit_fatal "unable to read from ${__shunit_script}" - __shunit_mode=${__SHUNIT_MODE_STANDALONE} -fi - -# Create a temporary storage location. -__shunit_tmpDir=`_shunit_mktempDir` - -# Provide a public temporary directory for unit test scripts. -# TODO(kward): document this. -SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" -command mkdir "${SHUNIT_TMPDIR}" - -# Setup traps to clean up after ourselves. -trap '_shunit_cleanup EXIT' 0 -trap '_shunit_cleanup INT' 2 -trap '_shunit_cleanup TERM' 15 - -# Create phantom functions to work around issues with Cygwin. -_shunit_mktempFunc -PATH="${__shunit_tmpDir}:${PATH}" - -# Make sure phantom functions are executable. This will bite if `/tmp` (or the -# current `$TMPDIR`) points to a path on a partition that was mounted with the -# 'noexec' option. The noexec command was created with `_shunit_mktempFunc()`. -noexec 2>/dev/null || _shunit_fatal \ - 'Please declare TMPDIR with path on partition with exec permission.' - -# We must manually source the tests in standalone mode. -if command [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then - # shellcheck disable=SC1090 - command . "`_shunit_prepForSourcing \"${__shunit_script}\"`" -fi - -# Configure default output coloring behavior. -_shunit_configureColor "${SHUNIT_COLOR}" - -# Execute the oneTimeSetUp function (if it exists). -oneTimeSetUp -command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_fatal "oneTimeSetUp() returned non-zero return code." - -# Command line selected tests or suite selected tests -if command [ "$#" -ge 2 ]; then - # Argument $1 is either the filename of tests or '--'; either way, skip it. - shift - # Remaining arguments ($2 .. $#) are assumed to be test function names. - # Interate through all remaining args in "$@" in a POSIX (likely portable) way. - # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop - for _shunit_arg_ do - suite_addTest "${_shunit_arg_}" - done - unset _shunit_arg_ -else - # Execute the suite function defined in the parent test script. - # DEPRECATED as of 2.1.0. - suite -fi - -# If no tests or suite specified, dynamically build a list of functions. -if command [ -z "${__shunit_suite}" ]; then - shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` - for shunit_func_ in ${shunit_funcs_}; do - suite_addTest "${shunit_func_}" - done -fi -unset shunit_func_ shunit_funcs_ - -# Execute the suite of unit tests. -_shunit_execSuite - -# Execute the oneTimeTearDown function (if it exists). -oneTimeTearDown -command [ $? -eq ${SHUNIT_TRUE} ] \ - || _shunit_fatal "oneTimeTearDown() returned non-zero return code." - -# Generate a report summary. -_shunit_generateReport - -# That's it folks. -command [ "${__shunit_testsFailed}" -eq 0 ] -exit $? diff --git a/external/yq/scripts/test-docker.sh b/external/yq/scripts/test-docker.sh deleted file mode 100755 index 427233c..0000000 --- a/external/yq/scripts/test-docker.sh +++ /dev/null @@ -1,5 +0,0 @@ -#! /bin/bash -set -e - -docker build . -t temp -docker run --rm -it --entrypoint sh temp -c 'touch a' diff --git a/external/yq/scripts/test.sh b/external/yq/scripts/test.sh deleted file mode 100755 index a18392c..0000000 --- a/external/yq/scripts/test.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test') diff --git a/external/yq/scripts/xcompile.sh b/external/yq/scripts/xcompile.sh deleted file mode 100755 index bf19cda..0000000 --- a/external/yq/scripts/xcompile.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -e -# you may need to go install github.com/mitchellh/gox@v1.0.1 first - -CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 darwin/arm64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64" - -cd build - -find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh -tar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh - -rm yq_windows_386.exe.tar.gz -rm yq_windows_amd64.exe.tar.gz - -zip yq_windows_386.zip yq_windows_386.exe -zip yq_windows_amd64.zip yq_windows_amd64.exe - -rm yq.1 - -rhash -r -a . -o checksums - -rhash --list-hashes > checksums_hashes_order - -cp ../scripts/extract-checksum.sh . \ No newline at end of file diff --git a/external/yq/snap/snapcraft.yaml b/external/yq/snap/snapcraft.yaml deleted file mode 100755 index ec50ab9..0000000 --- a/external/yq/snap/snapcraft.yaml +++ /dev/null @@ -1,22 +0,0 @@ -name: yq -version: '4.30.1' -summary: A lightweight and portable command-line YAML processor -description: | - The aim of the project is to be the jq or sed of yaml files. - -base: core18 -grade: stable # devel|stable. must be 'stable' to release into candidate/stable channels -confinement: strict - -apps: - yq: - command: yq - plugs: [home] - -parts: - yq: - plugin: go - go-channel: 1.19/stable - source: . - source-type: git - go-importpath: github.com/mikefarah/yq diff --git a/external/yq/test/utils.go b/external/yq/test/utils.go deleted file mode 100755 index f270c5b..0000000 --- a/external/yq/test/utils.go +++ /dev/null @@ -1,72 +0,0 @@ -package test - -import ( - "bufio" - "bytes" - "fmt" - "os" - "reflect" - "testing" - - "github.com/pkg/diff" - "github.com/pkg/diff/write" - yaml "gopkg.in/yaml.v3" -) - -func ParseData(rawData string) yaml.Node { - var parsedData yaml.Node - err := yaml.Unmarshal([]byte(rawData), &parsedData) - if err != nil { - fmt.Printf("Error parsing yaml: %v\n", err) - os.Exit(1) - } - return parsedData -} - -func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) { - opts := []write.Option{write.TerminalColor()} - var differenceBuffer bytes.Buffer - expectedString := fmt.Sprintf("%v", expectedValue) - actualString := fmt.Sprintf("%v", actualValue) - if err := diff.Text("expected", "actual", expectedString, actualString, bufio.NewWriter(&differenceBuffer), opts...); err != nil { - t.Error(err) - } else { - t.Error(differenceBuffer.String()) - } -} - -func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) { - t.Helper() - if expectedValue != actualValue { - printDifference(t, expectedValue, actualValue) - } -} - -func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) { - t.Helper() - if !reflect.DeepEqual(expectedValue, actualValue) { - printDifference(t, expectedValue, actualValue) - } -} - -func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { - t.Helper() - if !reflect.DeepEqual(expectedValue, actualValue) { - t.Error(context) - printDifference(t, expectedValue, actualValue) - } -} - -func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { - t.Helper() - opts := []write.Option{write.TerminalColor()} - if expectedValue != actualValue { - t.Error(context) - var differenceBuffer bytes.Buffer - if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil { - t.Error(err) - } else { - t.Error(differenceBuffer.String()) - } - } -} diff --git a/external/yq/utf8.csv b/external/yq/utf8.csv deleted file mode 100755 index d1503b8..0000000 --- a/external/yq/utf8.csv +++ /dev/null @@ -1,3 +0,0 @@ -id,first,last -1,john,smith -1,jane,smith diff --git a/external/yq/yq.go b/external/yq/yq.go deleted file mode 100755 index 239ff0f..0000000 --- a/external/yq/yq.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "os" - - command "github.com/mikefarah/yq/v4/cmd" -) - -func main() { - cmd := command.New() - - args := os.Args[1:] - - _, _, err := cmd.Find(args) - if err != nil && args[0] != "__complete" { - // default command when nothing matches... - newArgs := []string{"eval"} - cmd.SetArgs(append(newArgs, os.Args[1:]...)) - - } - - if err := cmd.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/external/yq/yq_linux_amd64 b/external/yq/yq_linux_amd64 deleted file mode 100755 index 29fdd62..0000000 Binary files a/external/yq/yq_linux_amd64 and /dev/null differ diff --git a/nodes/orchestration/CMakeLists.txt b/nodes/orchestration/CMakeLists.txt index 84e3bdf..a6af870 100755 --- a/nodes/orchestration/CMakeLists.txt +++ b/nodes/orchestration/CMakeLists.txt @@ -152,6 +152,13 @@ execute_process ( ) install(FILES ${z} DESTINATION orchestration/lib) +execute_process ( + COMMAND sh -c "which yq" + OUTPUT_VARIABLE yq + OUTPUT_STRIP_TRAILING_WHITESPACE +) +install(FILES ${yq} DESTINATION orchestration/scripts RENAME yq PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) + install(DIRECTORY scripts DESTINATION orchestration USE_SOURCE_PERMISSIONS) gen_help( diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 0221c3f..5daea51 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Copyright Check Point Software Technologies LTD +# Copyright open-appsec Software Technologies LTD FILESYSTEM_PATH="/etc/cp" LOG_FILE_PATH="/var/log" SMB_LOG_FILE_PATH="/storage" @@ -256,7 +256,7 @@ while true; do var_fog_address=$1 elif [ "$1" = "--max-log-size-kb" ]; then shift - WATCHDOG_MAX_FILE_SIZE=$1 + WATCHDOG_MAX_FILE_SIZE=$(($1 * 1024)) elif [ "$1" = "--max-log-rotation" ]; then shift WATCHDOG_MAX_ROTATIONS=$1 @@ -307,7 +307,7 @@ while true; do echo "Log files path: ${LOG_FILE_PATH}" elif [ "$1" = "--arm64_trustbox" ] || [ "$1" = "--arm64_linaro" ] || [ "$1" = "--arm32_rpi" ] || [ "$1" = "--gaia" ] || [ "$1" = "--smb_mrv_v1" ] || [ "$1" = "--smb_sve_v2" ] || [ "$1" = "--smb_thx_v3" ] || [ "$1" = "--x86" ] || [ "$1" = "./orchestration_package.sh" ]; then shift - continue + continue elif [ "$1" = "--skip_registration" ]; then var_skip_registration=true elif echo "$1" | grep -q ${FORCE_CLEAN_FLAG}; then @@ -536,7 +536,7 @@ install_watchdog() cp_copy service/x86/ubuntu16/nano_agent.service /etc/systemd/system/nano_agent.service echo "ExecStart=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/nano_agent.service echo "ExecStartPost=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" >> /etc/systemd/system/nano_agent.service - echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/nano_agent.service + echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/nano_agent.service cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" @@ -673,7 +673,7 @@ upgrade_conf_if_needed() [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && . "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') - if ! [ -z "$previous_mode" ]; then + if ! [ -z "$previous_mode" ]; then var_orchestration_mode=${previous_mode} fi @@ -716,12 +716,10 @@ copy_orchestration_executable() cp_copy open-appsec-cloud-mgmt-k8s ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-cloud-mgmt-k8s cp_copy open-appsec-ctl.sh ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-ctl.sh - if [ $var_hybrid_mode = true ]; then - if [ -f /ext/appsec/local_policy.yaml ]; then - cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml" - else - cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml - fi + if [ -f /ext/appsec/local_policy.yaml ]; then + cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml" + else + cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml fi } @@ -729,9 +727,9 @@ copy_k8s_executable() { if [ "$IS_K8S_ENV" = "true" ]; then cp -f k8s-check-update-listener.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - cp -f k8s-check-update-trigger.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh - chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh + chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh + cp -f k8s-check-update-trigger.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh + chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh fi } @@ -950,18 +948,18 @@ install_orchestration() echo "MAX_FILE_SIZE=${WATCHDOG_MAX_FILE_SIZE}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} echo "MAX_ROTATION=${WATCHDOG_MAX_ROTATIONS}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - if [ -n "${FILESYSTEM_PATH}" ]; then - echo "CP_ENV_FILESYSTEM=${FILESYSTEM_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${LOG_FILE_PATH}" ]; then - echo "CP_ENV_LOG_FILE=${LOG_FILE_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${USR_LIB_PATH}" ]; then - echo "CP_USR_LIB_PATH=${USR_LIB_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${INIT_D_PATH}" ]; then - echo "CP_INIT_D_PATH=${INIT_D_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi + if [ -n "${FILESYSTEM_PATH}" ]; then + echo "CP_ENV_FILESYSTEM=${FILESYSTEM_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${LOG_FILE_PATH}" ]; then + echo "CP_ENV_LOG_FILE=${LOG_FILE_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${USR_LIB_PATH}" ]; then + echo "CP_USR_LIB_PATH=${USR_LIB_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${INIT_D_PATH}" ]; then + echo "CP_INIT_D_PATH=${INIT_D_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi fi if [ -z "${var_token}" ] && [ ${var_hybrid_mode} = false ] && [ ${var_offline_mode} = false ] && [ -z ${EGG_MODE} ] && [ ${var_no_otp} = false ]; then