Repository: mikefarah/yq Branch: master Commit: 34d3a2930855 Files: 536 Total size: 1.5 MB Directory structure: gitextract_0o7jw5o7/ ├── .dockerignore ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report_v4.md │ │ └── feature_request.md │ ├── dependabot.yml │ ├── instructions/ │ │ └── instructions.md │ └── workflows/ │ ├── codeql.yml │ ├── docker-release.yml │ ├── go.yml │ ├── release.yml │ └── snap-release.yml ├── .gitignore ├── .golangci.bck.yml ├── .golangci.yml ├── .goreleaser.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Dockerfile ├── Dockerfile.dev ├── LICENSE ├── Makefile ├── Makefile.variables ├── README.md ├── acceptance_tests/ │ ├── bad_args.sh │ ├── basic.sh │ ├── completion.sh │ ├── empty.sh │ ├── flags.sh │ ├── front-matter.sh │ ├── header-processing-off.sh │ ├── inputs-format-auto.sh │ ├── inputs-format.sh │ ├── leading-separator.sh │ ├── load-file.sh │ ├── nul-separator.sh │ ├── output-format.sh │ ├── pipe.sh │ ├── pretty-print.sh │ ├── shebang.sh │ └── split-printer.sh ├── action.yml ├── agents.md ├── cmd/ │ ├── completion.go │ ├── constant.go │ ├── evaluate_all_command.go │ ├── evaluate_all_command_test.go │ ├── evaluate_sequence_command.go │ ├── evaluate_sequence_command_test.go │ ├── root.go │ ├── root_test.go │ ├── unwrap_flag.go │ ├── utils.go │ ├── utils_test.go │ ├── version.go │ └── version_test.go ├── cspell.config.yaml ├── examples/ │ ├── array.yaml │ ├── bad.yaml │ ├── base64.txt │ ├── data.lua │ ├── data1-no-comments.yaml │ ├── data1.yaml │ ├── data2.yaml │ ├── data3.yaml │ ├── empty-no-comment.yaml │ ├── empty.yaml │ ├── environment.yq │ ├── example.properties │ ├── front-matter.yaml │ ├── instruction_sample.yaml │ ├── kyaml.yml │ ├── leading-separator.yaml │ ├── merge-anchor.yaml │ ├── mike.xml │ ├── mike2.xml │ ├── multiline-text.yaml │ ├── multiple_docs.yaml │ ├── multiple_docs_small.yaml │ ├── numbered_keys.yml │ ├── order.yaml │ ├── order.yml │ ├── sample.hcl │ ├── sample.ini │ ├── sample.json │ ├── sample.tf │ ├── sample.toml │ ├── sample.yaml │ ├── sample2.hcl │ ├── sample_array.yaml │ ├── sample_array_2.yaml │ ├── sample_no_sections.ini │ ├── sample_objects.csv │ ├── sample_text.yaml │ ├── simple-anchor-exploded.yaml │ ├── simple-anchor.yaml │ ├── small.properties │ ├── small.xml │ ├── small.yaml │ └── thing.yml ├── github-action/ │ ├── Dockerfile │ └── entrypoint.sh ├── go.mod ├── go.sum ├── go_install_test.go ├── how-it-works.md ├── mkdocs.yml ├── pkg/ │ └── yqlib/ │ ├── all_at_once_evaluator.go │ ├── all_at_once_evaluator_test.go │ ├── base64_test.go │ ├── candidate_node.go │ ├── candidate_node_goccy_yaml.go │ ├── candidate_node_test.go │ ├── candidate_node_yaml.go │ ├── candidiate_node_json.go │ ├── chown_linux.go │ ├── chown_linux_test.go │ ├── chown_not_linux_os.go │ ├── color_print.go │ ├── color_print_test.go │ ├── context.go │ ├── context_test.go │ ├── csv.go │ ├── csv_test.go │ ├── data_tree_navigator.go │ ├── data_tree_navigator_test.go │ ├── decoder.go │ ├── decoder_base64.go │ ├── decoder_csv_object.go │ ├── decoder_goccy_yaml.go │ ├── decoder_hcl.go │ ├── decoder_ini.go │ ├── decoder_json.go │ ├── decoder_lua.go │ ├── decoder_properties.go │ ├── decoder_test.go │ ├── decoder_toml.go │ ├── decoder_uri.go │ ├── decoder_uri_test.go │ ├── decoder_xml.go │ ├── decoder_yaml.go │ ├── doc/ │ │ ├── .gitignore │ │ ├── notification-snippet.md │ │ ├── operators/ │ │ │ ├── add.md │ │ │ ├── alternative-default-value.md │ │ │ ├── anchor-and-alias-operators.md │ │ │ ├── array-to-map.md │ │ │ ├── assign-update.md │ │ │ ├── boolean-operators.md │ │ │ ├── collect-into-array.md │ │ │ ├── column.md │ │ │ ├── comment-operators.md │ │ │ ├── compare.md │ │ │ ├── contains.md │ │ │ ├── create-collect-into-object.md │ │ │ ├── datetime.md │ │ │ ├── delete.md │ │ │ ├── divide.md │ │ │ ├── document-index.md │ │ │ ├── encode-decode.md │ │ │ ├── entries.md │ │ │ ├── env-variable-operators.md │ │ │ ├── equals.md │ │ │ ├── error.md │ │ │ ├── eval.md │ │ │ ├── file-operators.md │ │ │ ├── filter.md │ │ │ ├── first.md │ │ │ ├── flatten.md │ │ │ ├── group-by.md │ │ │ ├── has.md │ │ │ ├── headers/ │ │ │ │ ├── Main.md │ │ │ │ ├── add.md │ │ │ │ ├── alternative-default-value.md │ │ │ │ ├── anchor-and-alias-operators.md │ │ │ │ ├── array-to-map.md │ │ │ │ ├── assign-update.md │ │ │ │ ├── boolean-operators.md │ │ │ │ ├── collect-into-array.md │ │ │ │ ├── column.md │ │ │ │ ├── comment-operators.md │ │ │ │ ├── compare.md │ │ │ │ ├── contains.md │ │ │ │ ├── create-collect-into-object.md │ │ │ │ ├── datetime.md │ │ │ │ ├── delete.md │ │ │ │ ├── divide.md │ │ │ │ ├── document-index.md │ │ │ │ ├── encode-decode.md │ │ │ │ ├── entries.md │ │ │ │ ├── env-variable-operators.md │ │ │ │ ├── equals.md │ │ │ │ ├── error.md │ │ │ │ ├── eval.md │ │ │ │ ├── file-operators.md │ │ │ │ ├── filter.md │ │ │ │ ├── first.md │ │ │ │ ├── flatten.md │ │ │ │ ├── group-by.md │ │ │ │ ├── has.md │ │ │ │ ├── keys.md │ │ │ │ ├── kind.md │ │ │ │ ├── length.md │ │ │ │ ├── line.md │ │ │ │ ├── load.md │ │ │ │ ├── map.md │ │ │ │ ├── max.md │ │ │ │ ├── min.md │ │ │ │ ├── modulo.md │ │ │ │ ├── multiply-merge.md │ │ │ │ ├── omit.md │ │ │ │ ├── parent.md │ │ │ │ ├── path.md │ │ │ │ ├── pick.md │ │ │ │ ├── pipe.md │ │ │ │ ├── pivot.md │ │ │ │ ├── recursive-descent-glob.md │ │ │ │ ├── reduce.md │ │ │ │ ├── reverse.md │ │ │ │ ├── select.md │ │ │ │ ├── shuffle.md │ │ │ │ ├── slice-array.md │ │ │ │ ├── sort-keys.md │ │ │ │ ├── sort.md │ │ │ │ ├── split-into-documents.md │ │ │ │ ├── string-operators.md │ │ │ │ ├── style.md │ │ │ │ ├── subtract.md │ │ │ │ ├── tag.md │ │ │ │ ├── to_number.md │ │ │ │ ├── traverse-read.md │ │ │ │ ├── union.md │ │ │ │ ├── unique.md │ │ │ │ ├── variable-operators.md │ │ │ │ └── with.md │ │ │ ├── keys.md │ │ │ ├── kind.md │ │ │ ├── length.md │ │ │ ├── line.md │ │ │ ├── load.md │ │ │ ├── map.md │ │ │ ├── max.md │ │ │ ├── min.md │ │ │ ├── modulo.md │ │ │ ├── multiply-merge.md │ │ │ ├── omit.md │ │ │ ├── parent.md │ │ │ ├── path.md │ │ │ ├── pick.md │ │ │ ├── pipe.md │ │ │ ├── pivot.md │ │ │ ├── recursive-descent-glob.md │ │ │ ├── reduce.md │ │ │ ├── reverse.md │ │ │ ├── select.md │ │ │ ├── shuffle.md │ │ │ ├── slice-array.md │ │ │ ├── sort-keys.md │ │ │ ├── sort.md │ │ │ ├── split-into-documents.md │ │ │ ├── string-operators.md │ │ │ ├── style.md │ │ │ ├── subtract.md │ │ │ ├── tag.md │ │ │ ├── to_number.md │ │ │ ├── traverse-read.md │ │ │ ├── union.md │ │ │ ├── unique.md │ │ │ ├── variable-operators.md │ │ │ └── with.md │ │ └── usage/ │ │ ├── base64.md │ │ ├── convert.md │ │ ├── csv-tsv.md │ │ ├── formatting-expressions.md │ │ ├── hcl.md │ │ ├── headers/ │ │ │ ├── base64.md │ │ │ ├── convert.md │ │ │ ├── csv-tsv.md │ │ │ ├── formatting-expressions.md │ │ │ ├── hcl.md │ │ │ ├── kyaml.md │ │ │ ├── properties.md │ │ │ ├── recipes.md │ │ │ ├── toml.md │ │ │ └── xml.md │ │ ├── kyaml.md │ │ ├── lua.md │ │ ├── properties.md │ │ ├── recipes.md │ │ ├── shellvariables.md │ │ ├── toml.md │ │ └── xml.md │ ├── encoder.go │ ├── encoder_base64.go │ ├── encoder_csv.go │ ├── encoder_hcl.go │ ├── encoder_ini.go │ ├── encoder_json.go │ ├── encoder_kyaml.go │ ├── encoder_lua.go │ ├── encoder_properties.go │ ├── encoder_properties_test.go │ ├── encoder_sh.go │ ├── encoder_shellvariables.go │ ├── encoder_shellvariables_test.go │ ├── encoder_test.go │ ├── encoder_toml.go │ ├── encoder_uri.go │ ├── encoder_xml.go │ ├── encoder_yaml.go │ ├── expression_parser.go │ ├── expression_parser_test.go │ ├── expression_postfix.go │ ├── expression_processing_test.go │ ├── file_utils.go │ ├── format.go │ ├── format_test.go │ ├── formatting_expressions_test.go │ ├── front_matter.go │ ├── front_matter_test.go │ ├── goccy_yaml_test.go │ ├── hcl.go │ ├── hcl_test.go │ ├── ini.go │ ├── ini_test.go │ ├── json.go │ ├── json_test.go │ ├── kyaml.go │ ├── kyaml_test.go │ ├── lexer.go │ ├── lexer_participle.go │ ├── lexer_participle_test.go │ ├── lib.go │ ├── lib_test.go │ ├── lua.go │ ├── lua_test.go │ ├── matchKeyString.go │ ├── matchKeyString_test.go │ ├── no_base64.go │ ├── no_csv.go │ ├── no_hcl.go │ ├── no_ini.go │ ├── no_json.go │ ├── no_kyaml.go │ ├── no_lua.go │ ├── no_props.go │ ├── no_sh.go │ ├── no_shellvariables.go │ ├── no_toml.go │ ├── no_uri.go │ ├── no_xml.go │ ├── operation.go │ ├── operator_add.go │ ├── operator_add_test.go │ ├── operator_alternative.go │ ├── operator_alternative_test.go │ ├── operator_anchors_aliases.go │ ├── operator_anchors_aliases_test.go │ ├── operator_array_to_map_test.go │ ├── operator_assign.go │ ├── operator_assign_test.go │ ├── operator_booleans.go │ ├── operator_booleans_test.go │ ├── operator_collect.go │ ├── operator_collect_object.go │ ├── operator_collect_object_test.go │ ├── operator_collect_test.go │ ├── operator_column.go │ ├── operator_column_test.go │ ├── operator_comments.go │ ├── operator_comments_test.go │ ├── operator_compare.go │ ├── operator_contains.go │ ├── operator_contains_test.go │ ├── operator_create_map.go │ ├── operator_create_map_test.go │ ├── operator_datetime.go │ ├── operator_datetime_test.go │ ├── operator_delete.go │ ├── operator_delete_test.go │ ├── operator_divide.go │ ├── operator_divide_test.go │ ├── operator_document_index.go │ ├── operator_document_index_test.go │ ├── operator_encoder_decoder.go │ ├── operator_encoder_decoder_test.go │ ├── operator_entries.go │ ├── operator_entries_test.go │ ├── operator_env.go │ ├── operator_env_test.go │ ├── operator_equals.go │ ├── operator_equals_test.go │ ├── operator_error.go │ ├── operator_error_test.go │ ├── operator_eval.go │ ├── operator_eval_test.go │ ├── operator_expression.go │ ├── operator_file.go │ ├── operator_file_test.go │ ├── operator_filter.go │ ├── operator_filter_test.go │ ├── operator_first.go │ ├── operator_first_test.go │ ├── operator_flatten.go │ ├── operator_flatten_test.go │ ├── operator_group_by.go │ ├── operator_group_by_test.go │ ├── operator_has.go │ ├── operator_has_test.go │ ├── operator_keys.go │ ├── operator_keys_test.go │ ├── operator_kind.go │ ├── operator_kind_test.go │ ├── operator_length.go │ ├── operator_length_test.go │ ├── operator_line.go │ ├── operator_line_test.go │ ├── operator_load.go │ ├── operator_load_test.go │ ├── operator_map.go │ ├── operator_map_test.go │ ├── operator_modulo.go │ ├── operator_modulo_test.go │ ├── operator_multiply.go │ ├── operator_multiply_test.go │ ├── operator_omit.go │ ├── operator_omit_test.go │ ├── operator_parent.go │ ├── operator_parent_test.go │ ├── operator_path.go │ ├── operator_path_test.go │ ├── operator_pick.go │ ├── operator_pick_test.go │ ├── operator_pipe.go │ ├── operator_pipe_test.go │ ├── operator_pivot.go │ ├── operator_pivot_test.go │ ├── operator_recursive_descent.go │ ├── operator_recursive_descent_test.go │ ├── operator_reduce.go │ ├── operator_reduce_test.go │ ├── operator_reverse.go │ ├── operator_reverse_test.go │ ├── operator_select.go │ ├── operator_select_test.go │ ├── operator_self.go │ ├── operator_shuffle.go │ ├── operator_shuffle_test.go │ ├── operator_slice.go │ ├── operator_slice_test.go │ ├── operator_sort.go │ ├── operator_sort_keys.go │ ├── operator_sort_keys_test.go │ ├── operator_sort_test.go │ ├── operator_split_document.go │ ├── operator_split_document_test.go │ ├── operator_strings.go │ ├── operator_strings_test.go │ ├── operator_style.go │ ├── operator_style_test.go │ ├── operator_subtract.go │ ├── operator_subtract_test.go │ ├── operator_tag.go │ ├── operator_tag_test.go │ ├── operator_to_number.go │ ├── operator_to_number_test.go │ ├── operator_traverse_path.go │ ├── operator_traverse_path_test.go │ ├── operator_union.go │ ├── operator_union_test.go │ ├── operator_unique.go │ ├── operator_unique_test.go │ ├── operator_value.go │ ├── operator_value_test.go │ ├── operator_variables.go │ ├── operator_variables_test.go │ ├── operator_with.go │ ├── operator_with_test.go │ ├── operators.go │ ├── operators_compare_test.go │ ├── operators_test.go │ ├── printer.go │ ├── printer_node_info.go │ ├── printer_node_info_test.go │ ├── printer_test.go │ ├── printer_writer.go │ ├── properties.go │ ├── properties_test.go │ ├── recipes_test.go │ ├── security_prefs.go │ ├── shellvariables.go │ ├── shellvariables_test.go │ ├── stream_evaluator.go │ ├── string_evaluator.go │ ├── string_evaluator_test.go │ ├── toml.go │ ├── toml_test.go │ ├── utils.go │ ├── write_in_place_handler.go │ ├── write_in_place_handler_test.go │ ├── xml.go │ ├── xml_test.go │ ├── yaml.go │ └── yaml_test.go ├── project-words.txt ├── release_instructions.txt ├── release_notes.txt ├── scripts/ │ ├── acceptance.sh │ ├── build-small-yq.sh │ ├── build-tinygo-yq.sh │ ├── bump-version.sh │ ├── check.sh │ ├── compare-jq.sh │ ├── compare-versions-output.sh │ ├── copy-docs.sh │ ├── coverage.sh │ ├── devtools.sh │ ├── extract-checksum.sh │ ├── format.sh │ ├── generate-man-page-md.sh │ ├── generate-man-page.sh │ ├── install-man-page.sh │ ├── release-deb.sh │ ├── secure.sh │ ├── setup.sh │ ├── shunit2 │ ├── spelling.sh │ ├── test-docker.sh │ ├── test.sh │ └── xcompile.sh ├── snap/ │ └── snapcraft.yaml ├── test/ │ ├── format_test.go │ └── utils.go ├── test.yq ├── utf8.csv ├── yq.go └── yq_test.go ================================================ FILE CONTENTS ================================================ ================================================ FILE: .dockerignore ================================================ bin/* ================================================ FILE: .github/FUNDING.yml ================================================ github: mikefarah ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report_v4.md ================================================ --- name: Bug report - V4 about: Create a report to help us improve title: '' labels: bug, v4 assignees: '' --- **Describe the bug** A clear and concise description of what the bug is. Note that any how to questions should be posted in the discussion board and not raised as an issue. Version of yq: 4.X.X Operating system: mac/linux/windows/.... Installed via: docker/binary release/homebrew/snap/... **Input Yaml** Concise yaml document(s) (as simple as possible to show the bug, please keep it to 10 lines or less) data1.yml: ```yaml this: should really work ``` data2.yml: ```yaml but: it strangely didn't ``` **Command** The command you ran: ``` yq eval-all 'select(fileIndex==0) | .a.b.c' data1.yml data2.yml ``` **Actual behaviour** ```yaml cat: meow ``` **Expected behaviour** ```yaml this: should really work but: it strangely didn't ``` **Additional context** Add any other context about the problem here. ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.md ================================================ --- name: Feature request - V4 about: Suggest an idea for this project title: '' labels: enhancement, v4 assignees: '' --- **Please describe your feature request.** A clear and concise description of what the request is and what it would solve. Eg. I wish I could use yq to [...] Note: - how to questions should be posted in the discussion board and not raised as an issue. - V3 will no longer have any enhancements. **Describe the solution you'd like** If we have data1.yml like: (please keep to around 10 lines ) ```yaml country: Australia ``` And we run a command: ```bash yq 'predictWeatherOf(.country)' ``` it could output ```yaml temp: 32 ``` **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. ================================================ FILE: .github/dependabot.yml ================================================ # To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: docker directory: / schedule: day: thursday interval: weekly - package-ecosystem: github-actions directory: / schedule: day: thursday interval: weekly - package-ecosystem: gomod directory: / schedule: day: thursday interval: weekly ================================================ FILE: .github/instructions/instructions.md ================================================ When you find a bug - make sure to include a new test that exposes the bug, as well as the fix for the bug itself. ================================================ FILE: .github/workflows/codeql.yml ================================================ # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '24 3 * * 1' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'go' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: - name: Checkout repository uses: actions/checkout@v6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v4 # ℹ️ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 ================================================ FILE: .github/workflows/docker-release.yml ================================================ name: Release Docker on: release: types: [released] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: publishDocker: environment: dockerhub env: IMAGE_NAME: mikefarah/yq runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - name: Set up QEMU uses: docker/setup-qemu-action@v4 with: platforms: all - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v3 with: version: latest - name: Available platforms run: echo ${{ steps.buildx.outputs.platforms }} && docker version - name: Login to Docker Hub uses: docker/login-action@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Login to GitHub Container Registry uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push image run: | echo "GithubRef: ${GITHUB_REF}" VERSION=${GITHUB_REF##*/} echo "VERSION: ${VERSION}" IMAGE_VERSION=${VERSION:1} echo "IMAGE_VERSION: ${IMAGE_VERSION}" PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x" echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}" docker buildx build \ --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \ --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \ --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \ --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \ --label "org.opencontainers.image.licenses=MIT" \ --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \ --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \ --label "org.opencontainers.image.title=yq" \ --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \ --label "org.opencontainers.image.version=${IMAGE_VERSION}" \ --platform "${PLATFORMS}" \ --pull \ --push \ -t "${IMAGE_NAME}:${IMAGE_VERSION}" \ -t "${IMAGE_NAME}:4" \ -t "${IMAGE_NAME}:latest" \ -t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}" \ -t "ghcr.io/${IMAGE_NAME}:4" \ -t "ghcr.io/${IMAGE_NAME}:latest" \ . cd github-action docker buildx build \ --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \ --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \ --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \ --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \ --label "org.opencontainers.image.licenses=MIT" \ --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \ --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \ --label "org.opencontainers.image.title=yq" \ --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \ --label "org.opencontainers.image.version=${IMAGE_VERSION}" \ --platform "${PLATFORMS}" \ --pull \ --push \ -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \ -t "${IMAGE_NAME}:4-githubaction" \ -t "${IMAGE_NAME}:latest-githubaction" \ -t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \ -t "ghcr.io/${IMAGE_NAME}:4-githubaction" \ -t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \ . ================================================ FILE: .github/workflows/go.yml ================================================ name: Build on: [push, pull_request] permissions: contents: read jobs: build: name: Build runs-on: ubuntu-latest steps: - name: Set up Go uses: actions/setup-go@v6 with: go-version: '^1.20' id: go - name: Check out code into the Go module directory uses: actions/checkout@v6 - name: Get dependencies run: | go get -v -t -d ./... if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh dep ensure fi - name: Check the build shell: bash -l {0} run: | export PATH=${PATH}:`go env GOPATH`/bin scripts/devtools.sh make local build ================================================ FILE: .github/workflows/release.yml ================================================ name: Release YQ on: push: tags: - 'v4.*' - 'draft-*' jobs: publishGitRelease: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: actions/setup-go@v6 with: go-version: '^1.20' check-latest: true - name: Compile man page markup id: gen-man-page-md run: | ./scripts/generate-man-page-md.sh - name: Get the version id: get_version run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}" - name: Generate man page uses: docker://pandoc/core:2.14.2 id: gen-man-page with: args: >- --standalone --to man --variable=title:"YQ" --variable=section:"1" --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}" --variable=author:"Mike Farah" --output=yq.1 man.md - name: Cross compile run: | sudo apt-get install rhash -y go install github.com/goreleaser/goreleaser/v2@latest ./scripts/xcompile.sh - name: Release uses: softprops/action-gh-release@v1 with: files: build/* draft: true fail_on_unmatched_files: true ================================================ FILE: .github/workflows/snap-release.yml ================================================ name: Release Snap on: release: types: [released] # Allows you to run this workflow manually from the Actions tab workflow_dispatch: jobs: buildSnap: environment: snap runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - uses: snapcore/action-build@v1 id: build env: SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} with: snapcraft-args: "remote-build --launchpad-accept-public-upload" - uses: snapcore/action-publish@v1 env: SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }} with: snap: ${{ steps.build.outputs.snap }} release: stable ================================================ FILE: .gitignore ================================================ # Compiled Object files, Static and Dynamic libs (Shared Objects) *.o *.a *.so # Folders _obj _test bin build build-done .DS_Store # Architecture specific extensions/prefixes *.[568vq] [568vq].out *.cgo1.go *.cgo2.c _cgo_defun.c _cgo_gotypes.go _cgo_export.* _testmain.go cover.out coverage.out coverage.html coverage_sorted.txt *.exe *.test *.prof yaml vendor/ tmp/ cover/ yq # snapcraft parts/ prime/ .snapcraft/ yq*.snap test.yml test*.yml test*.tf test*.xml test*.toml test*.yaml *.kyaml test_dir1/ test_dir2/ 0.yml 1.yml 2.yml # man page man.md yq.1 # debian pkg _build debian/files # intellij /.idea # vscode .vscode yq3 # Golang .gomodcache/ .gocache/ ================================================ FILE: .golangci.bck.yml ================================================ run: timeout: 5m linters: enable: - asciicheck - depguard - errorlint - gci - gochecknoinits - gofmt - goimports - gosec - gosimple - staticcheck - unused - misspell - nakedret - nolintlint - predeclared - revive - unconvert - unparam linters-settings: depguard: rules: prevent_unmaintained_packages: list-mode: lax files: - $all - "!$test" deny: - pkg: io/ioutil desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil" issues: exclude-rules: - linters: - revive text: "var-naming" ================================================ FILE: .golangci.yml ================================================ version: "2" linters: enable: - asciicheck - depguard - errorlint - gochecknoinits - gosec - misspell - nakedret - nolintlint - predeclared - revive - unconvert - unparam settings: misspell: locale: UK ignore-rules: - color - colors depguard: rules: prevent_unmaintained_packages: list-mode: lax files: - $all - '!$test' deny: - pkg: io/ioutil desc: 'replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil' exclusions: generated: lax presets: - comments - common-false-positives - legacy - std-error-handling rules: - linters: - revive text: var-naming paths: - third_party$ - builtin$ - examples$ formatters: enable: - gci - gofmt - goimports exclusions: generated: lax paths: - third_party$ - builtin$ - examples$ ================================================ FILE: .goreleaser.yaml ================================================ version: 2 dist: build builds: - id: yq binary: yq_{{ .Os }}_{{ .Arch }} ldflags: - -s -w env: - CGO_ENABLED=0 targets: - darwin_amd64 - darwin_arm64 - freebsd_386 - freebsd_amd64 - freebsd_arm - linux_386 - linux_amd64 - linux_arm - linux_arm64 - linux_loong64 - linux_mips - linux_mips64 - linux_mips64le - linux_mipsle - linux_ppc64 - linux_ppc64le - linux_riscv64 - linux_s390x - netbsd_386 - netbsd_amd64 - netbsd_arm - openbsd_386 - openbsd_amd64 - windows_386 - windows_amd64 - windows_arm64 no_unique_dist_dir: true release: disable: true skip_upload: true ================================================ FILE: CODE_OF_CONDUCT.md ================================================ # Contributor Covenant Code of Conduct ## Our Pledge In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards Examples of behaviour that contributes to creating a positive environment include: * Using welcoming and inclusive language * Being respectful of differing viewpoints and experiences * Gracefully accepting constructive criticism * Focusing on what is best for the community * Showing empathy towards other community members Examples of unacceptable behaviour by participants include: * The use of sexualized language or imagery and unwelcome sexual attention or advances * Trolling, insulting/derogatory comments, and personal or political attacks * Public or private harassment * Publishing others' private information, such as a physical or electronic address, without explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Our Responsibilities Project maintainers are responsible for clarifying the standards of acceptable behaviour and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behaviour. Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviours that they deem inappropriate, threatening, offensive, or harmful. ## Scope This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behaviour may be reported by contacting the project team at mikefarah@gmail.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq ================================================ FILE: CONTRIBUTING.md ================================================ # Before you begin Not all new PRs will be merged in It's recommended to check with the owner first (e.g. raise an issue) to discuss a new feature before developing, to ensure your hard efforts don't go to waste. PRs to fix bugs and issues are almost always welcome :pray: please ensure you write tests as well. The following types of PRs will _not_ be accepted: - **Significant refactors** take a lot of time to understand and can have all sorts of unintended side effects. If you think there's a better way to do things (that requires significant changes) raise an issue for discussion first :) - **Release pipeline PRs** are a security risk - it's too easy for a serious vulnerability to sneak in (either intended or not). If there is a new cool way of releasing things, raise an issue for discussion first - it will need to be gone over with a fine tooth comb. - **Version bumps** are handled by dependabot, the bot will auto-raise PRs and they will be regularly merged in. - **New release platforms** At this stage, yq is not going to maintain any other release platforms other than GitHub and Docker - that said, I'm more than happy to put in other community maintained methods in the README for visibility :heart: # Development ## Initial Setup 1. Install [Golang](https://golang.org/) (version 1.24.0 or later) 2. Run `scripts/devtools.sh` to install required development tools: - golangci-lint for code linting - gosec for security analysis 3. Run `make [local] vendor` to install vendor dependencies 4. Run `make [local] test` to ensure you can run the existing tests ## Development Workflow 1. **Write unit tests first** - Changes will not be accepted without corresponding unit tests (see Testing section below) 2. **Make your code changes** 3. **Run tests and linting**: `make [local] test` (this runs formatting, linting, security checks, and tests) 4. **Create your PR** and get kudos! :) ## Make Commands - Use `make [local] ` for local development (runs in Docker container) - Use `make ` for CI/CD environments - Common commands: - `make [local] vendor` - Install dependencies - `make [local] test` - Run all checks and tests - `make [local] build` - Build the yq binary - `make [local] format` - Format code - `make [local] check` - Run linting and security checks # Code Quality ## Linting and Formatting The project uses strict linting rules defined in `.golangci.yml`. All code must pass: - **Code formatting**: gofmt, goimports, gci - **Linting**: revive, errorlint, gosec, misspell, and others - **Security checks**: gosec security analysis - **Spelling checks**: misspell detection Run `make [local] check` to verify your code meets all quality standards. ## Code Style Guidelines - Follow standard Go conventions - Use meaningful variable names - Add comments for public functions and complex logic - Keep functions focused and reasonably sized - Use the project's existing patterns and conventions # Testing ## Test Structure Tests in yq use the `expressionScenario` pattern. Each test scenario includes: - `expression`: The yq expression to test - `document`: Input YAML/JSON (optional) - `expected`: Expected output - `skipDoc`: Whether to skip documentation generation ## Writing Tests 1. **Find the appropriate test file** (e.g., `operator_add_test.go` for addition operations) 2. **Add your test scenario** to the `*OperatorScenarios` slice 3. **Run the specific test**: `go test -run TestAddOperatorScenarios` (replace with appropriate test name) 4. **Verify documentation generation** (see Documentation section) ## Test Examples ```go var addOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: `"foo" + "bar"`, expected: []string{ "D0, P[], (!!str)::foobar\n", }, }, { document: "apples: 3", expression: `.apples + 3`, expected: []string{ "D0, P[apples], (!!int)::6\n", }, }, } ``` ## Running Tests - **All tests**: `make [local] test` - **Specific test**: `go test -run TestName` - **With coverage**: `make [local] cover` # Documentation ## Documentation Generation The project uses a documentation system that combines static headers with dynamically generated content from tests. ### How It Works 1. **Static headers** are defined in `pkg/yqlib/doc/operators/headers/*.md` 2. **Dynamic content** is generated from test scenarios in `*_test.go` files 3. **Generated docs** are created in `pkg/yqlib/doc/*.md` by concatenating headers with test-generated content 4. **Documentation is synced** to the gitbook branch for the website ### Updating Operator Documentation #### For Test-Generated Documentation Most operator documentation is generated from tests. To update: 1. **Find the test file** (e.g., `operator_add_test.go`) 2. **Update test scenarios** - each `expressionScenario` with `skipDoc: false` becomes documentation 3. **Run the test** to regenerate docs: ```bash cd pkg/yqlib go test -run TestAddOperatorScenarios ``` 4. **Verify the generated documentation** in `pkg/yqlib/doc/add.md` 5. **Create a PR** with your changes #### For Header-Only Documentation If documentation exists only in `headers/*.md` files: 1. **Update the header file directly** (e.g., `pkg/yqlib/doc/operators/headers/add.md`) 2. **Create a PR** with your changes ### Updating Static Documentation For documentation not in the master branch: 1. **Check the gitbook branch** for additional pages 2. **Update the `*.md` files** directly 3. **Create a PR** to the gitbook branch ### Documentation Best Practices - **Write clear, concise examples** in test scenarios - **Use meaningful variable names** in examples - **Include edge cases** and error conditions - **Test your documentation changes** by running the specific test - **Verify generated output** matches expectations Note: PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama). # Troubleshooting ## Common Setup Issues ### Docker/Podman Issues - **Problem**: `make` commands fail with Docker errors - **Solution**: Ensure Docker or Podman is running and accessible - **Alternative**: Use `make local ` to run in containers ### Go Version Issues - **Problem**: Build fails with Go version errors - **Solution**: Ensure you have Go 1.24.0 or later installed - **Check**: Run `go version` to verify ### Vendor Dependencies - **Problem**: `make vendor` fails or dependencies are outdated - **Solution**: ```bash go mod tidy make [local] vendor ``` ### Linting Failures - **Problem**: `make check` fails with linting errors - **Solution**: ```bash make [local] format # Auto-fix formatting # Manually fix remaining linting issues make [local] check # Verify fixes ``` ### Test Failures - **Problem**: Tests fail locally but pass in CI - **Solution**: ```bash make [local] test # Run in Docker container ``` - **Problem**: Tests fail with a VCS error: ```bash error obtaining VCS status: exit status 128 Use -buildvcs=false to disable VCS stamping. ``` - **Solution**: Git security mechanisms prevent Golang from detecting the Git details inside the container; either build with the `local` option, or pass GOFLAGS to disable Golang buildvcs behaviour. ```bash make local test # OR make test GOFLAGS='-buildvcs=true' ``` ### Documentation Generation Issues - **Problem**: Generated docs don't update after test changes - **Solution**: ```bash cd pkg/yqlib go test -run TestSpecificOperatorScenarios # Check if generated file updated in pkg/yqlib/doc/ ``` ## Getting Help - **Check existing issues**: Search GitHub issues for similar problems - **Create an issue**: If you can't find a solution, create a detailed issue - **Ask questions**: Use GitHub Discussions for general questions - **Join the community**: Check the project's community channels ================================================ FILE: Dockerfile ================================================ FROM golang:1.26.1 AS builder WORKDIR /go/src/mikefarah/yq COPY . . RUN CGO_ENABLED=0 go build -ldflags "-s -w" . # RUN ./scripts/test.sh -- this too often times out in the github pipeline. RUN ./scripts/acceptance.sh # Choose alpine as a base image to make this useful for CI, as many # CI tools expect an interactive shell inside the container FROM alpine:3 AS production LABEL maintainer="Mike Farah " COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq WORKDIR /workdir RUN set -eux; \ addgroup -g 1000 yq; \ adduser -u 1000 -G yq -s /bin/sh -h /home/yq -D yq RUN chown -R yq:yq /workdir USER yq ENTRYPOINT ["/usr/bin/yq"] ================================================ FILE: Dockerfile.dev ================================================ FROM golang:1.26.1 RUN apt-get update && \ apt-get install -y npm && \ npm install -g npx cspell@latest COPY scripts/devtools.sh /opt/devtools.sh RUN set -e -x && \ /opt/devtools.sh ENV PATH=/go/bin:$PATH ENV CGO_ENABLED 0 ENV GOPATH /go:/yq ================================================ FILE: LICENSE ================================================ The MIT License (MIT) Copyright (c) 2017 Mike Farah Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ================================================ FILE: Makefile ================================================ MAKEFLAGS += --warn-undefined-variables SHELL := /bin/bash .SHELLFLAGS := -o pipefail -euc .DEFAULT_GOAL := install ENGINE := $(shell { (podman version > /dev/null 2>&1 && command -v podman) || command -v docker; } 2>/dev/null) include Makefile.variables .PHONY: help help: @echo 'Management commands for cicdtest:' @echo @echo 'Usage:' @echo ' ## Develop / Test Commands' @echo ' make build Build yq binary.' @echo ' make install Install yq.' @echo ' make xcompile Build cross-compiled binaries of yq.' @echo ' make vendor Install dependencies to vendor directory.' @echo ' make format Run code formatter.' @echo ' make check Run static code analysis (lint).' @echo ' make secure Run gosec.' @echo ' make test Run tests on project.' @echo ' make cover Run tests and capture code coverage metrics on project.' @echo ' make clean Clean the directory tree of produced artifacts.' @echo @echo ' ## Utility Commands' @echo ' make setup Configures Minishfit/Docker directory mounts.' @echo .PHONY: clean clean: @rm -rf bin build cover *.out ## prefix before other make targets to run in your local dev environment local: | quiet @$(eval ENGINERUN= ) @$(eval GOFLAGS="$(GOFLAGS)" ) @mkdir -p tmp @touch tmp/dev_image_id quiet: # this is silly but shuts up 'Nothing to be done for `local`' @: prepare: tmp/dev_image_id tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh @mkdir -p tmp @${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true @${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev . @${ENGINE} inspect -f "{{ .ID }}" ${DEV_IMAGE} > tmp/dev_image_id # ---------------------------------------------- # build .PHONY: build build: build/dev .PHONY: build/dev build/dev: test *.go @mkdir -p bin/ ${ENGINERUN} go build --ldflags "$(LDFLAGS)" ${ENGINERUN} bash ./scripts/acceptance.sh ## Compile the project for multiple OS and Architectures. xcompile: check @rm -rf build/ @mkdir -p build ${ENGINERUN} bash ./scripts/xcompile.sh @find build -type d -exec chmod 755 {} \; || : @find build -type f -exec chmod 755 {} \; || : .PHONY: install install: build ${ENGINERUN} go install # Each of the fetch should be an entry within vendor.json; not currently included within project .PHONY: vendor vendor: tmp/dev_image_id @mkdir -p vendor ${ENGINERUN} go mod vendor # ---------------------------------------------- # develop and test .PHONY: format format: vendor ${ENGINERUN} bash ./scripts/format.sh .PHONY: spelling spelling: format ${ENGINERUN} bash ./scripts/spelling.sh .PHONY: secure secure: spelling ${ENGINERUN} bash ./scripts/secure.sh .PHONY: check check: secure ${ENGINERUN} bash ./scripts/check.sh .PHONY: test test: check ${ENGINERUN} bash ./scripts/test.sh .PHONY: cover cover: check @rm -rf cover/ @mkdir -p cover ${ENGINERUN} bash ./scripts/coverage.sh @find cover -type d -exec chmod 755 {} \; || : @find cover -type f -exec chmod 644 {} \; || : .PHONY: release release: xcompile ${ENGINERUN} bash ./scripts/publish.sh # ---------------------------------------------- # utilities .PHONY: setup setup: @bash ./scripts/setup.sh ================================================ FILE: Makefile.variables ================================================ export PROJECT = yq IMPORT_PATH := github.com/mikefarah/${PROJECT} export GIT_COMMIT = $(shell git rev-parse --short HEAD) export GIT_DIRTY = $(shell test -n "$$(git status --porcelain)" && echo "+CHANGES" || true) export GIT_DESCRIBE = $(shell git describe --tags --always) GOFLAGS := LDFLAGS := LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY} LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE} LDFLAGS += -w -s GITHUB_TOKEN ?= # Windows environment? CYG_CHECK := $(shell hash cygpath 2>/dev/null && echo 1) ifeq ($(CYG_CHECK),1) VBOX_CHECK := $(shell hash VBoxManage 2>/dev/null && echo 1) # Docker Toolbox (pre-Windows 10) ifeq ($(VBOX_CHECK),1) ROOT := /${PROJECT} else # Docker Windows ROOT := $(shell cygpath -m -a "$(shell pwd)") endif else # all non-windows environments ROOT := $(shell pwd) # Deliberately use `command -v` instead of `which` to be POSIX compliant SELINUX := $(shell command -v getenforce >/dev/null 2>&1 && echo :z) endif DEV_IMAGE := ${PROJECT}_dev ENGINERUN := ${ENGINE} run --rm \ -e LDFLAGS="${LDFLAGS}" \ -e GOFLAGS="${GOFLAGS}" \ -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ -v ${ROOT}/vendor:/go/src${SELINUX} \ -v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH}${SELINUX} \ -w /${PROJECT}/src/${IMPORT_PATH} \ ${DEV_IMAGE} ================================================ FILE: README.md ================================================ # yq ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg) A lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) (a popular JSON processor) like syntax but works with yaml files as well as json, kyaml, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. yq is written in Go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below. ## Quick Usage Guide ### Basic Operations **Read a value:** ```bash yq '.a.b[0].c' file.yaml ``` **Pipe from STDIN:** ```bash yq '.a.b[0].c' < file.yaml ``` **Update a yaml file in place:** ```bash yq -i '.a.b[0].c = "cool"' file.yaml ``` **Update using environment variables:** ```bash NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml ``` ### Advanced Operations **Merge multiple files:** ```bash # merge two files yq -n 'load("file1.yaml") * load("file2.yaml")' # merge using globs (note: `ea` evaluates all files at once instead of in sequence) yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml ``` **Multiple updates to a yaml file:** ```bash yq -i ' .a.b[0].c = "cool" | .x.y.z = "foobar" | .person.name = strenv(NAME) ' file.yaml ``` **Find and update an item in an array:** ```bash # Note: requires input file - add your file at the end yq -i '(.[] | select(.name == "foo") | .address) = "12 cat st"' data.yaml ``` **Convert between formats:** ```bash # Convert JSON to YAML (pretty print) yq -Poy sample.json # Convert YAML to JSON yq -o json file.yaml # Convert XML to YAML yq -o yaml file.xml ``` See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information. Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell) ## Install ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest) ### wget Use wget to download pre-compiled binaries. Choose your platform and architecture: **For Linux (example):** ```bash # Set your platform variables (adjust as needed) VERSION=v4.2.0 PLATFORM=linux_amd64 # Download compressed binary wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM}.tar.gz -O - |\ tar xz && sudo mv yq_${PLATFORM} /usr/local/bin/yq # Or download plain binary wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM} -O /usr/local/bin/yq &&\ chmod +x /usr/local/bin/yq ``` **Latest version (Linux AMD64):** ```bash wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq &&\ chmod +x /usr/local/bin/yq ``` **Available platforms:** `linux_amd64`, `linux_arm64`, `linux_arm`, `linux_386`, `darwin_amd64`, `darwin_arm64`, `windows_amd64`, `windows_386`, etc. ### MacOS / Linux via Homebrew: Using [Homebrew](https://brew.sh/) ``` brew install yq ``` ### Linux via snap: ``` snap install yq ``` #### Snap notes `yq` installs with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can: ``` sudo cat /etc/myfile | yq '.a.path' ``` And to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge): ``` sudo cat /etc/myfile | yq '.a.path = "value"' | sudo sponge /etc/myfile ``` or write to a temporary file: ``` sudo cat /etc/myfile | yq '.a.path = "value"' | sudo tee /etc/myfile.tmp sudo mv /etc/myfile.tmp /etc/myfile rm /etc/myfile.tmp ``` ### Run with Docker or Podman #### One-time use: ```bash # Docker - process files in current directory docker run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml # Podman - same usage as Docker podman run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml ``` **Security note:** You can run `yq` in Docker with restricted privileges: ```bash docker run --rm --security-opt=no-new-privileges --cap-drop all --network none \ -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml ``` #### Pipe data via STDIN: You'll need to pass the `-i --interactive` flag to Docker/Podman: ```bash # Process piped data docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml # Same with Podman podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml ``` #### Run commands interactively: ```bash docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq ``` ```bash podman run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq ``` It can be useful to have a bash function to avoid typing the whole docker command: ```bash yq() { docker run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@" } ``` ```bash yq() { podman run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@" } ``` #### Running as root: `yq`'s container image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the container image, or you're having permissions issues when attempting to read/write files you'll need to either: ``` docker run --user="root" -it --entrypoint sh mikefarah/yq ``` ``` podman run --user="root" -it --entrypoint sh mikefarah/yq ``` Or, in your Dockerfile: ``` FROM mikefarah/yq USER root RUN apk add --no-cache bash USER yq ``` #### Missing timezone data By default, the alpine image yq uses does not include timezone data. If you'd like to use the `tz` operator, you'll need to include this data: ``` FROM mikefarah/yq USER root RUN apk add --no-cache tzdata USER yq ``` #### Podman with SELinux If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount: ``` -v "${PWD}":/workdir:z ``` ### GitHub Action ``` - name: Set foobar to cool uses: mikefarah/yq@master with: cmd: yq -i '.foo.bar = "cool"' 'config.yml' - name: Get an entry with a variable that might contain dots or spaces id: get_username uses: mikefarah/yq@master with: cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml - name: Reuse a variable obtained in another step run: echo ${{ steps.get_username.outputs.result }} ``` See https://mikefarah.gitbook.io/yq/usage/github-action for more. ### Go Install: ``` go install github.com/mikefarah/yq/v4@latest ``` ## Community Supported Installation methods As these are supported by the community :heart: - however, they may be out of date with the officially supported releases. _Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._ ### X-CMD Checkout `yq` on x-cmd: https://x-cmd.com/mod/yq - Instant Results: See the output of your yq filter in real-time. - Error Handling: Encounter a syntax error? It will display the error message and the results of the closest valid filter Thanks @edwinjhlee! ### Nix ``` nix profile install nixpkgs#yq-go ``` See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go) ### Webi ``` webi yq ``` See [webi](https://webinstall.dev/) Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq) ### Arch Linux ``` pacman -S go-yq ``` ### Windows: Using [Chocolatey](https://chocolatey.org) [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq) [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq) ``` choco install yq ``` Supported by @chillum (https://chocolatey.org/packages/yq) Using [scoop](https://scoop.sh/) ``` scoop install main/yq ``` Using [winget](https://learn.microsoft.com/en-us/windows/package-manager/) ``` winget install --id MikeFarah.yq ``` ### MacPorts: Using [MacPorts](https://www.macports.org/) ``` sudo port selfupdate sudo port install yq ``` Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot) ### Alpine Linux Alpine Linux v3.20+ (and Edge): ``` apk add yq-go ``` Alpine Linux up to v3.19: ``` apk add yq ``` Supported by Tuan Hoang (https://pkgs.alpinelinux.org/packages?name=yq-go) ### Flox: Flox can be used to install yq on Linux, MacOS, and Windows through WSL. ``` flox install yq ``` ### MacOS / Linux via gah: Using [gah](https://github.com/marverix/gah) ``` gah install yq ``` ## Features - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/) - Written in portable go, so you can download a lovely dependency free binary - Uses similar syntax as `jq` but works with YAML, INI, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files - Fully supports multi document yaml files - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble) - Colorized yaml output - [Date/Time manipulation and formatting with TZ](https://mikefarah.gitbook.io/yq/operators/datetime) - [Deep data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) - [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators). - [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map) - Keeps yaml formatting and comments when updating (though there are issues with whitespace) - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode) - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load) - [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert) - [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml) - [Convert to/from hcl (terraform)](https://mikefarah.gitbook.io/yq/v/v4.x/usage/hcl) - [Convert to/from toml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/toml) - [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) - [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv) - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion) - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things. - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus) ## [Usage](https://mikefarah.gitbook.io/yq/) Check out the [documentation](https://mikefarah.gitbook.io/yq/) for more detailed and advanced usage. ``` Usage: yq [flags] yq [command] Examples: # yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN) # Use the '-p/--input-format' flag to specify a format type. cat file.xml | yq -p xml # read the "stuff" node from "myfile.yml" yq '.stuff' < myfile.yml # update myfile.yml in place yq -i '.stuff = "foo"' myfile.yml # print contents of sample.json as idiomatic YAML yq -P -oy sample.json Available Commands: completion Generate the autocompletion script for the specified shell eval (default) Apply the expression to each document in each yaml file in sequence eval-all Loads _all_ yaml documents of _all_ yaml files and runs expression once help Help about any command Flags: -C, --colors force print with colors --csv-auto-parse parse CSV YAML/JSON values (default true) --csv-separator char CSV Separator character (default ,) --debug-node-info debug node info -e, --exit-status set exit status if there are no matches or null or false is returned --expression string forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file. --from-file string Load expression from specified file. -f, --front-matter string (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact --header-preprocess Slurp any header comments and separators before processing expression. (default true) -h, --help help for yq -I, --indent int sets indent level for output (default 2) -i, --inplace update the file in place of first file given. -p, --input-format string [auto|a|yaml|y|json|j|kyaml|ky|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|lua|l|ini|i] parse format for input. (default "auto") --lua-globals output keys as top-level global variables --lua-prefix string prefix (default "return ") --lua-suffix string suffix (default ";\n") --lua-unquoted output unquoted string keys (e.g. {foo="bar"}) -M, --no-colors force print with no colors -N, --no-doc Don't print document separators (---) -0, --nul-output Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char. -n, --null-input Don't read input, simply evaluate the expression given. Useful for creating docs from scratch. -o, --output-format string [auto|a|yaml|y|json|j|kyaml|ky|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|hcl|h|shell|s|lua|l|ini|i] output format type. (default "auto") -P, --prettyPrint pretty print, shorthand for '... style = ""' --properties-array-brackets use [x] in array paths (e.g. for SpringBoot) --properties-separator string separator to use between keys and values (default " = ") --security-disable-env-ops Disable env related operations. --security-disable-file-ops Disable file related operations (e.g. load) --shell-key-separator string separator for shell variable key paths (default "_") -s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created. --split-exp-file string Use a file to specify the split-exp expression. --string-interpolation Toggles strings interpolation of \(exp) (default true) --tsv-auto-parse parse TSV YAML/JSON values (default true) -r, --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true) -v, --verbose verbose mode -V, --version Print version information and quit --xml-attribute-prefix string prefix for xml attributes (default "+@") --xml-content-name string name for xml content (if no attribute name is present). (default "+content") --xml-directive-name string name for xml directives (e.g. ) (default "+directive") --xml-keep-namespace enables keeping namespace after parsing attributes (default true) --xml-proc-inst-prefix string prefix for xml processing instructions (e.g. ) (default "+p_") --xml-raw-token enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details. (default true) --xml-skip-directives skip over directives (e.g. ) --xml-skip-proc-inst skip over process instructions (e.g. ) --xml-strict-mode enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details. --yaml-fix-merge-anchor-to-spec Fix merge anchor to match YAML spec. Will default to true in late 2025 Use "yq [command] --help" for more information about a command. ``` ## Troubleshooting ### Common Issues **PowerShell quoting issues:** ```powershell # Use single quotes for expressions yq '.a.b[0].c' file.yaml # Or escape double quotes yq ".a.b[0].c = \"value\"" file.yaml ``` ### Getting Help - **Check existing issues**: [GitHub Issues](https://github.com/mikefarah/yq/issues) - **Ask questions**: [GitHub Discussions](https://github.com/mikefarah/yq/discussions) - **Documentation**: [Complete documentation](https://mikefarah.gitbook.io/yq/) - **Examples**: [Recipes and examples](https://mikefarah.gitbook.io/yq/recipes) ## Known Issues / Missing Features - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details) - Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell) - "yes", "no" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes. See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions. ================================================ FILE: acceptance_tests/bad_args.sh ================================================ #!/bin/bash testWriteInPlacePipeIn() { result=$(./yq e -i -n '.a' 2>&1) assertEquals 1 $? assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result" } testWriteInPlacePipeInEvalall() { result=$(./yq ea -i -n '.a' 2>&1) assertEquals 1 $? assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result" } testWriteInPlaceWithSplit() { result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1) assertEquals 1 $? assertEquals "Error: write in place cannot be used with split file" "$result" } testWriteInPlaceWithSplitEvalAll() { result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1) assertEquals 1 $? assertEquals "Error: write in place cannot be used with split file" "$result" } testNullWithFiles() { result=$(./yq e -n '.a = "thing"' test.yml 2>&1) assertEquals 1 $? assertEquals "Error: cannot pass files in when using null-input flag" "$result" } testNullWithFilesEvalAll() { result=$(./yq ea -n '.a = "thing"' test.yml 2>&1) assertEquals 1 $? assertEquals "Error: cannot pass files in when using null-input flag" "$result" } source ./scripts/shunit2 ================================================ FILE: acceptance_tests/basic.sh ================================================ #!/bin/bash setUp() { rm test*.yml 2>/dev/null || true rm .xyz 2>/dev/null || true rm instructions.txt 2>/dev/null || true } testBasicEvalRoundTrip() { ./yq -n ".a = 123" > test.yml X=$(./yq '.a' test.yml) assertEquals 123 "$X" } testBasicTrailingContent() { cat >test-trailing.yml <test-trailing.yml <test-trailing.yml <test-trailing.yml < test.yml X=$(cat test.yml | ./yq '.') assertEquals "a: 123" "$X" } testBasicExpressionMatchesFileName() { ./yq -n ".xyz = 123" > test.yml touch .xyz X=$(./yq --expression '.xyz' test.yml) assertEquals "123" "$X" X=$(./yq ea --expression '.xyz' test.yml) assertEquals "123" "$X" } testBasicExpressionFromFile() { ./yq -n ".xyz = 123" > test.yml echo '.xyz = "meow" | .cool = "frog"' > instructions.txt X=$(./yq --from-file instructions.txt test.yml -o=j -I=0) assertEquals '{"xyz":"meow","cool":"frog"}' "$X" X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0) assertEquals '{"xyz":"meow","cool":"frog"}' "$X" } testBasicExpressionFromFileDos() { ./yq -n ".xyz = 123" > test.yml echo '.xyz = "meow" | .cool = "frog"' | sed 's/$'"/`echo \\\r`/" > instructions.txt X=$(./yq --from-file instructions.txt test.yml -o=j -I=0) assertEquals '{"xyz":"meow","cool":"frog"}' "$X" X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0) assertEquals '{"xyz":"meow","cool":"frog"}' "$X" } testBasicGitHubAction() { ./yq -n ".a = 123" > test.yml X=$(cat /dev/null | ./yq test.yml) assertEquals "a: 123" "$X" X=$(cat /dev/null | ./yq e test.yml) assertEquals "a: 123" "$X" X=$(cat /dev/null | ./yq ea test.yml) assertEquals "a: 123" "$X" } testBasicGitHubActionWithExpression() { ./yq -n ".a = 123" > test.yml X=$(cat /dev/null | ./yq '.a' test.yml) assertEquals "123" "$X" X=$(cat /dev/null | ./yq e '.a' test.yml) assertEquals "123" "$X" X=$(cat /dev/null | ./yq ea '.a' test.yml) assertEquals "123" "$X" } testBasicEvalAllAllFiles() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(./yq ea test.yml test2.yml) Y=$(./yq e '.' test.yml test2.yml) assertEquals "$Y" "$X" } # when given a file, don't read STDIN # otherwise strange things start happening # in scripts # https://github.com/mikefarah/yq/issues/1115 testBasicCatWithFilesNoDash() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq test2.yml) Y=$(./yq e '.' test2.yml) assertEquals "$Y" "$X" } # when the nullinput flag is used # don't automatically read STDIN (this breaks github actions) testBasicCreateFileGithubAction() { cat /dev/null | ./yq -n ".a = 123" > test.yml } testBasicEvalAllCatWithFilesNoDash() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq ea test2.yml) Y=$(./yq e '.' test2.yml) assertEquals "$Y" "$X" } testBasicCatWithFilesNoDashWithExp() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq '.a' test2.yml) Y=$(./yq e '.a' test2.yml) assertEquals "$Y" "$X" } testBasicEvalAllCatWithFilesNoDashWithExp() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq ea '.a' test2.yml) Y=$(./yq e '.a' test2.yml) assertEquals "$Y" "$X" } testBasicStdInWithFiles() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq - test2.yml) Y=$(./yq e '.' test.yml test2.yml) assertEquals "$Y" "$X" } testBasicEvalAllStdInWithFiles() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq ea - test2.yml) Y=$(./yq e '.' test.yml test2.yml) assertEquals "$Y" "$X" } testBasicStdInWithFilesReverse() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq test2.yml -) Y=$(./yq e '.' test2.yml test.yml) assertEquals "$Y" "$X" } testBasicEvalAllStdInWithFilesReverse() { ./yq -n ".a = 123" > test.yml ./yq -n ".a = 124" > test2.yml X=$(cat test.yml | ./yq ea test2.yml -) Y=$(./yq e '.' test2.yml test.yml) assertEquals "$Y" "$X" } testBasicEvalRoundTripNoEval() { ./yq -n ".a = 123" > test.yml X=$(./yq '.a' test.yml) assertEquals 123 "$X" } testBasicStdInWithOneArg() { ./yq e -n ".a = 123" > test.yml X=$(cat test.yml | ./yq e ".a") assertEquals 123 "$X" X=$(cat test.yml | ./yq ea ".a") assertEquals 123 "$X" X=$(cat test.yml | ./yq ".a") assertEquals 123 "$X" } testBasicUpdateInPlaceSequence() { cat >test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml < test.yml X=$(./yq e '.z' test.yml) assertEquals "null" "$X" } testBasicExitStatus() { echo "a: cat" > test.yml X=$(./yq e -e '.z' test.yml 2&>/dev/null) assertEquals 1 "$?" } testBasicExitStatusNoEval() { echo "a: cat" > test.yml X=$(./yq -e '.z' test.yml 2&>/dev/null) assertEquals 1 "$?" } testBasicExtractFieldWithSeparator() { cat >test.yml <test.yml <test.yml <test.yml <&1) assertEquals 0 $? assertContains "$result" "Completion ended with directive:" } source ./scripts/shunit2 ================================================ FILE: acceptance_tests/empty.sh ================================================ #!/bin/bash setUp() { rm test*.yml || true cat >test.yml <test.yml X=$(./yq e test.yml) expected=$(cat test.yml) assertEquals 0 $? assertEquals "$expected" "$X" } testEmptyEvalNoNewLineWithExpression() { echo -n "# comment" >test.yml X=$(./yq e '.apple = "tree"' test.yml) read -r -d '' expected << EOM # comment apple: tree EOM assertEquals "$expected" "$X" } testEmptyEvalPipe() { X=$(./yq e - < test.yml) assertEquals 0 $? } testEmptyCommentsWithExpressionEval() { read -r -d '' expected << EOM # comment apple: tree EOM X=$(./yq e '.apple="tree"' test.yml) assertEquals "$expected" "$X" } testEmptyCommentsWithExpressionEvalAll() { read -r -d '' expected << EOM # comment apple: tree EOM X=$(./yq ea '.apple="tree"' test.yml) assertEquals "$expected" "$X" } testEmptyWithExpressionEval() { rm test.yml touch test.yml expected="apple: tree" X=$(./yq e '.apple="tree"' test.yml) assertEquals "$expected" "$X" } testEmptyWithExpressionEvalAll() { rm test.yml touch test.yml expected="apple: tree" X=$(./yq ea '.apple="tree"' test.yml) assertEquals "$expected" "$X" } source ./scripts/shunit2 ================================================ FILE: acceptance_tests/flags.sh ================================================ #!/bin/bash setUp() { rm test*.yml || true cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml </dev/null || true rm test*.toml 2>/dev/null || true rm test*.tfstate 2>/dev/null || true rm test*.json 2>/dev/null || true rm test*.properties 2>/dev/null || true rm test*.csv 2>/dev/null || true rm test*.tsv 2>/dev/null || true rm test*.xml 2>/dev/null || true } testInputJson() { cat >test.json <test.toml <test.tfstate <test.json <test.properties <test.properties <test.csv <test.tsv <test.xml <BiBi EOL read -r -d '' expected << EOM BiBi EOM X=$(./yq e test.xml) assertEquals "$expected" "$X" X=$(./yq ea test.xml) assertEquals "$expected" "$X" } testInputXmlNamespaces() { cat >test.xml < EOL read -r -d '' expected << EOM EOM X=$(./yq e test.xml) assertEquals "$expected" "$X" X=$(./yq ea test.xml) assertEquals "$expected" "$X" } testInputXmlStrict() { cat >test.xml < ]> &writer;©right; EOL X=$(./yq --xml-strict-mode test.xml 2>&1) assertEquals 1 $? assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X" X=$(./yq ea --xml-strict-mode test.xml 2>&1) assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X" } testInputXmlGithubAction() { cat >test.xml <BiBi EOL read -r -d '' expected << EOM BiBi EOM X=$(cat /dev/null | ./yq e test.xml) assertEquals "$expected" "$X" X=$(cat /dev/null | ./yq ea test.xml) assertEquals "$expected" "$X" } source ./scripts/shunit2 ================================================ FILE: acceptance_tests/inputs-format.sh ================================================ #!/bin/bash setUp() { rm test*.yml 2>/dev/null || true rm test*.properties 2>/dev/null || true rm test*.csv 2>/dev/null || true rm test*.tsv 2>/dev/null || true rm test*.xml 2>/dev/null || true rm test*.tf 2>/dev/null || true } testInputProperties() { cat >test.properties <test.properties <test.csv <test.csv <test.csv <test.tsv <test.tsv <test.kyaml <<'EOL' # leading { a: 1, # a line # head b b: 2, c: [ # head d "d", # d line ], } EOL read -r -d '' expected <<'EOM' # leading a: 1 # a line # head b b: 2 c: # head d - d # d line EOM X=$(./yq e -p=kyaml -P test.kyaml) assertEquals "$expected" "$X" X=$(./yq ea -p=kyaml -P test.kyaml) assertEquals "$expected" "$X" } testInputXml() { cat >test.yml <BiBi EOL read -r -d '' expected << EOM cat: +content: BiBi +@legs: "4" EOM X=$(./yq e -p=xml test.yml) assertEquals "$expected" "$X" X=$(./yq ea -p=xml test.yml) assertEquals "$expected" "$X" } testInputXmlNamespaces() { cat >test.xml < EOL read -r -d '' expected << EOM +p_xml: version="1.0" map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@xsi:schemaLocation: some-url EOM X=$(./yq e -p=xml test.xml) assertEquals "$expected" "$X" X=$(./yq ea -p=xml test.xml) assertEquals "$expected" "$X" } testInputXmlRoundtrip() { cat >test.yml < Meow EOL read -r -d '' expected << EOM Meow EOM X=$(./yq -p=xml -o=xml test.yml) assertEquals "$expected" "$X" X=$(./yq ea -p=xml -o=xml test.yml) assertEquals "$expected" "$X" } testInputXmlStrict() { cat >test.yml < ]> &writer;©right; EOL X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1) assertEquals 1 $? assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X" X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1) assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X" } testInputXmlGithubAction() { cat >test.yml <BiBi EOL read -r -d '' expected << EOM cat: +content: BiBi +@legs: "4" EOM X=$(cat /dev/null | ./yq e -p=xml test.yml) assertEquals "$expected" "$X" X=$(cat /dev/null | ./yq ea -p=xml test.yml) assertEquals "$expected" "$X" } testInputTerraform() { cat >test.tf <test.tf <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <&1) assertEquals 1 $? assertEquals "Error: failed to load cat.yml: open cat.yml: no such file or directory" "$result" } testLoadFileExpNotExist() { result=$(./yq e -n 'load(.a)' 2>&1) assertEquals 1 $? assertEquals "Error: filename expression returned nil" "$result" } testStrLoadFileNotExist() { result=$(./yq e -n 'strload("cat.yml")' 2>&1) assertEquals 1 $? assertEquals "Error: failed to load cat.yml: open cat.yml: no such file or directory" "$result" } testStrLoadFileExpNotExist() { result=$(./yq e -n 'strload(.a)' 2>&1) assertEquals 1 $? assertEquals "Error: filename expression returned nil" "$result" } source ./scripts/shunit2 ================================================ FILE: acceptance_tests/nul-separator.sh ================================================ #!/bin/bash setUp() { rm test*.yml || true } ## Convenient bash shortcut to read records of NUL separated values ## from stdin the safe way. See example usage in the next tests. read-0() { local eof="" IFS='' while [ "$1" ]; do ## - The `-r` avoids bad surprise with '\n' and other interpreted ## sequences that can be read. ## - The `-d ''` is the (strange?) way to refer to NUL delimiter. ## - The `--` is how to avoid unpleasant surprises if your ## "$1" starts with "-" (minus) sign. This protection also ## will produce a readable error if you want to try to start ## your variable names with a "-". read -r -d '' -- "$1" || eof=1 shift done [ -z "$eof" ] ## fail on EOF } ## Convenient bash shortcut to be used with the next function `p-err` ## to read NUL separated values the safe way AND catch any errors from ## the process creating the stream of NUL separated data. See example ## usage in the tests. read-0-err() { local ret="$1" eof="" idx=0 last= read -r -- "${ret?}" <<<"0" shift while [ "$1" ]; do last=$idx read -r -d '' -- "$1" || { ## Put this last value in ${!ret} eof="$1" read -r -- "$ret" <<<"${!eof}" break } ((idx++)) shift done [ -z "$eof" ] || { if [ "$last" != 0 ]; then ## Uhoh, we have no idea if the errorlevel of the internal ## command was properly delimited with a NUL char, and ## anyway something went really wrong at least about the ## number of fields separated by NUL char and the one ## expected. echo "Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'" >&2 read -r -- "$ret" <<<"not-enough-values" else if ! [[ "${!ret}" =~ ^[0-9]+$ && "${!ret}" -ge 0 && "${!ret}" -le 127 ]]; then ## This could happen if you don't use `p-err` wrapper, ## or used stdout in unexpected ways in your inner ## command. echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2 read -r -- "$ret" <<<"last-value-not-a-number" fi fi false } } ## Simply runs command given as argument and adds errorlevel in the ## standard output. Is expected to be used in tandem with ## `read-0-err`. p-err() { local exp="$1" "$@" printf "%s" "$?" } wyq-r() { local exp="$1" ./yq e -0 -r=false "$1" printf "%s" "$?" } testBasicUsageRaw() { cat >test.yml < expected.out ## We need to compare binary content here. We have to filter the compared ## content through a representation that gets rid of NUL chars but accurately ## transcribe the content. ## Also as it would be nice to have a pretty output in case the test fails, ## we use here 'hd': a widely available shortcut to 'hexdump' that will ## pretty-print any binary to it's hexadecimal representation. ## ## Note that the standard `assertEquals` compare its arguments ## value, but they can't hold NUL characters (this comes from the ## limitation of the C API of `exec*(..)` functions that requires ## `const char *arv[]`). And these are NUL terminated strings. As a ## consequence, the NUL characters gets removed in bash arguments. assertEquals "$(hd expected.out)" \ "$(./yq e -0 '.a, .b' test.yml | hd)" rm expected.out } testBasicUsage() { local a b cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml < cat EOM X=$(./yq e --output-format=xml test.yml) assertEquals "$expected" "$X" X=$(./yq ea --output-format=xml test.yml) assertEquals "$expected" "$X" } testOutputXmlShort() { cat >test.yml < cat EOM X=$(./yq e --output-format=x test.yml) assertEquals "$expected" "$X" X=$(./yq ea --output-format=x test.yml) assertEquals "$expected" "$X" } testOutputKYaml() { cat >test.yml <<'EOL' # leading a: 1 # a line # head b b: 2 c: # head d - d # d line EOL read -r -d '' expected <<'EOM' # leading { a: 1, # a line # head b b: 2, c: [ # head d "d", # d line ], } EOM X=$(./yq e --output-format=kyaml test.yml) assertEquals "$expected" "$X" X=$(./yq ea --output-format=kyaml test.yml) assertEquals "$expected" "$X" } testOutputKYamlShort() { cat >test.yml <test.yml < cat dog EOM X=$(./yq e --output-format=x test.yml) assertEquals "$expected" "$X" X=$(./yq ea --output-format=x test.yml) assertEquals "$expected" "$X" } testLuaOutputPretty() { cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yq <test.yml <test.yml <test.yml <test.yml <test_splitExp.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test2.yml <test.yml <.go` implementing the `Encoder` interface: - `Encode()` - Convert a `CandidateNode` to your format and write to the output writer - `PrintDocumentSeparator()` - Handle document separators if your format requires them - `PrintLeadingContent()` - Handle leading content/comments if supported - `CanHandleAliases()` - Return whether your format supports YAML aliases See `encoder_json.go` or `encoder_base64.go` for examples. ### Step 2: Create the Decoder File Create `pkg/yqlib/decoder_.go` implementing the `Decoder` interface: - `Init()` - Initialize the decoder with the input reader and set up any needed state - `Decode()` - Decode one document from the input and return a `CandidateNode`, or `io.EOF` when finished See `decoder_json.go` or `decoder_base64.go` for examples. ### Step 3: Create Tests (Mandatory) Create a test file `pkg/yqlib/_test.go` using the `formatScenario` pattern: - Define test scenarios as `formatScenario` structs with fields: `description`, `input`, `expected`, `scenarioType` - `scenarioType` can be `"decode"` (test decoding to YAML) or `"roundtrip"` (encode/decode preservation) - Create a helper function `testScenario()` that switches on `scenarioType` - Create main test function `TestFormatScenarios()` that iterates over scenarios - The main test function should use `documentScenarios` to ensure testcase documentation is generated. Test coverage must include: - Basic data types (scalars, arrays, objects/maps) - Nested structures - Edge cases (empty inputs, special characters, escape sequences) - Format-specific features or syntax - Round-trip tests: decode → encode → decode should preserve data See `hcl_test.go` for a complete example. ### Step 4: Register the Format in format.go Edit `pkg/yqlib/format.go`: 1. Add a new format variable: - `""` is the formal name (e.g., "json", "yaml") - `[]string{...}` contains short aliases (can be empty) - The first function creates an encoder (can be nil for encode-only formats) - The second function creates a decoder (can be nil for decode-only formats) 2. Add the format to the `Formats` slice in the same file See existing formats in `format.go` for the exact structure. ### Step 5: Handle Encoder Configuration (if needed) If your format has preferences/configuration options: 1. Create a preferences struct with your configuration fields 2. Update the encoder to accept preferences in its factory function 3. Update `format.go` to pass the configured preferences 4. Update `operator_encoder_decoder.go` if special indent handling is needed (see existing formats like JSON and YAML for the pattern) This pattern is optional and only needed if your format has user-configurable options. ## Build Tags Use build tags to allow optional compilation of formats: - Add `//go:build !yq_no` at the top of your encoder and decoder files - Create a no-build version in `pkg/yqlib/no_.go` that returns nil for encoder/decoder factories This allows users to compile yq without certain formats using: `go build -tags yq_no` ## Working with CandidateNode The `CandidateNode` struct represents a YAML node with: - `Kind`: The node type (ScalarNode, SequenceNode, MappingNode) - `Tag`: The YAML tag (e.g., "!!str", "!!int", "!!map") - `Value`: The scalar value (for ScalarNode only) - `Content`: Child nodes (for SequenceNode and MappingNode) Key methods: - `node.guessTagFromCustomType()` - Infer the tag from Go type - `node.AsList()` - Convert to a list for processing - `node.CreateReplacement()` - Create a new replacement node - `NewCandidate()` - Create a new CandidateNode ## Key Points ✅ **DO:** - Implement only the `Encoder` and `Decoder` interfaces - Register your format in `format.go` only - Keep format-specific logic in your encoder/decoder files - Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles. - Use build tags for optional compilation - Add comprehensive tests - Run the specific encoder/decoder test (e.g. _test.go) whenever you make ay changes to the encoder_ or decoder_ - Handle errors gracefully - Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format. ❌ **DON'T:** - Modify `candidate_node.go` to add format-specific logic - Add format-specific fields to `CandidateNode` - Create special cases in core navigation or evaluation logic - Bypass the encoder/decoder interfaces - Use candidate_node tag attribute for anything other than indicate the data type ## Examples Refer to existing format implementations for patterns: - **Simple encoder/decoder**: `encoder_json.go`, `decoder_json.go` - **Complex with preferences**: `encoder_yaml.go`, `decoder_yaml.go` - **Encoder-only**: `encoder_sh.go` (ShFormat has nil decoder) - **String-only operations**: `encoder_base64.go`, `decoder_base64.go` ## Testing Your Implementation (Mandatory) Tests must be implemented in `_test.go` following the `formatScenario` pattern: 1. **Create test scenarios** using the `formatScenario` struct with fields: - `description`: Brief description of what's being tested - `input`: Sample input in your format - `expected`: Expected output (typically in YAML for decode tests) - `scenarioType`: Either `"decode"` or `"roundtrip"` 2. **Test coverage must include:** - Basic data types (scalars, arrays, objects/maps) - Nested structures - Edge cases (empty inputs, special characters, escape sequences) - Format-specific features or syntax - Round-trip tests: decode → encode → decode should preserve data 3. **Test function pattern:** - `testScenario()`: Helper function that switches on `scenarioType` - `TestFormatScenarios()`: Main test function that iterates over scenarios 4. **Example from existing formats:** - See `hcl_test.go` for a complete example - See `yaml_test.go` for YAML-specific patterns - See `json_test.go` for more complex scenarios ## Common Patterns ### Format with Indentation Use preferences to control output formatting: ```go type Preferences struct { Indent int } func (prefs *Preferences) Copy() Preferences { return *prefs } ``` ### Multiple Documents Decoders should support reading multiple documents: ```go func (dec *Decoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } // ... decode next document ... if noMoreDocuments { dec.finished = true } return candidate, nil } ``` --- # Adding a New Operator This guide explains how to add a new operator to yq. Operators are the core of yq's expression language and process `CandidateNode` objects without requiring modifications to `candidate_node.go` itself. ## Overview Operators transform data by implementing a handler function that processes a `Context` containing `CandidateNode` objects. Each operator is: 1. Defined as an `operationType` in `operation.go` 2. Registered in the lexer in `lexer_participle.go` 3. Implemented in its own `operator_.go` file 4. Tested in `operator__test.go` 5. Documented in `pkg/yqlib/doc/operators/headers/.md` ## Architecture ### Key Files - `pkg/yqlib/operation.go` - Defines `operationType` and operator registry - `pkg/yqlib/lexer_participle.go` - Registers operators with their syntax patterns - `pkg/yqlib/operator_.go` - Operator implementation - `pkg/yqlib/operator__test.go` - Operator tests using `expressionScenario` - `pkg/yqlib/doc/operators/headers/.md` - Documentation header ### Core Types **operationType:** ```go type operationType struct { Type string // Unique operator name (e.g., "REVERSE") NumArgs uint // Number of arguments (0 for no args) Precedence uint // Operator precedence (higher = higher precedence) Handler operatorHandler // The function that executes the operator CheckForPostTraverse bool // Whether to apply post-traversal logic ToString func(*Operation) string // Custom string representation } ``` **operatorHandler signature:** ```go type operatorHandler func(*dataTreeNavigator, Context, *ExpressionNode) (Context, error) ``` **expressionScenario for tests:** ```go type expressionScenario struct { description string subdescription string document string expression string expected []string skipDoc bool expectedError string } ``` ## Step-by-Step: Adding a New Operator ### Step 1: Create the Operator Implementation File Create `pkg/yqlib/operator_.go` implementing the operator handler function: - Implement the `operatorHandler` function signature - Process nodes from `context.MatchingNodes` - Return a new `Context` with results using `context.ChildContext()` - Use `candidate.CreateReplacement()` or `candidate.CreateReplacementWithComments()` to create new nodes - Handle errors gracefully with meaningful error messages See `operator_reverse.go` or `operator_keys.go` for examples. ### Step 2: Register the Operator in operation.go Add the operator type definition to `pkg/yqlib/operation.go`: ```go var OpType = &operationType{ Type: "", // All caps, matches pattern in lexer NumArgs: 0, // 0 for no args, 1+ for args Precedence: 50, // Typical range: 40-55 Handler: Operator, // Reference to handler function } ``` **Precedence guidelines:** - 10-20: Logical operators (OR, AND, UNION) - 30: Pipe operator - 40: Assignment and comparison operators - 42: Arithmetic operators (ADD, SUBTRACT, MULTIPLY, DIVIDE) - 50-52: Most other operators - 55: High precedence (e.g., GET_VARIABLE) **Optional fields:** - `CheckForPostTraverse: true` - If your operator can have another directly after it without the pipe character. Most of the time this is false. - `ToString: customToString` - Custom string representation (rarely needed) ### Step 3: Register the Operator in lexer_participle.go Edit `pkg/yqlib/lexer_participle.go` to add the operator to the lexer rules: - Use `simpleOp()` for simple keyword patterns - Use object syntax for regex patterns or complex syntax - Support optional characters with `_?` and aliases with `|` See existing operators in `lexer_participle.go` for pattern examples. ### Step 4: Create Tests (Mandatory) Create `pkg/yqlib/operator__test.go` using the `expressionScenario` pattern: - Define test scenarios with `description`, `document`, `expression`, and `expected` fields - `expected` is a slice of strings showing output format: `"D, P[], ()::\n"` - Set `skipDoc: true` for edge cases you don't want in generated documentation - Include `subdescription` for longer test names - Set `expectedError` if testing error cases - Create main test function that iterates over scenarios - The main test function should use `documentScenarios` to ensure testcase documentation is generated. Test coverage must include: - Basic data types and nested structures - Edge cases (empty inputs, special characters, type errors) - Multiple outputs if applicable - Format-specific features See `operator_reverse_test.go` for a simple example and `operator_keys_test.go` for complex cases. ### Step 5: Create Documentation Header Create `pkg/yqlib/doc/operators/headers/.md`: - Use the exact operator name as the title - Include a concise 1-2 sentence summary - Add additional context or examples if the operator is complex See existing headers in `doc/operators/headers/` for examples. ## Working with Context and CandidateNode ### Context Management - `context.ChildContext(results)` - Create child context with results - `context.GetVariable("varName")` - Get variables stored in context - `context.SetVariable("varName", value)` - Set variables in context ### CandidateNode Operations - `candidate.CreateReplacement(ScalarNode, "!!str", stringValue)` - Create a replacement node - `candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style)` - With style preserved - `candidate.Kind` - The node type (ScalarNode, SequenceNode, MappingNode) - `candidate.Tag` - The YAML tag (!!str, !!int, etc.) - `candidate.Value` - The scalar value (for ScalarNode only) - `candidate.Content` - Child nodes (for SequenceNode and MappingNode) - `candidate.guessTagFromCustomType()` - Infer the tag from Go type - `candidate.AsList()` - Convert to a list representation ## Key Points ✅ **DO:** - Implement the operator handler with the correct signature - Register in `operation.go` with appropriate precedence - Add the lexer pattern in `lexer_participle.go` - Write comprehensive tests covering normal and edge cases - Create a documentation header in `doc/operators/headers/` - Use `Context.ChildContext()` for proper context threading - Handle all node types gracefully - Return meaningful error messages ❌ **DON'T:** - Modify `candidate_node.go` (operators shouldn't need this) - Modify core navigation or evaluation logic - Bypass the handler function pattern - Add format-specific or operator-specific fields to `CandidateNode` - Skip tests or documentation ## Examples Refer to existing operator implementations for patterns: - **No-argument operator**: `operator_reverse.go` - Processes arrays/sequences - **Single-argument operator**: `operator_map.go` - Takes an expression argument - **Complex multi-output**: `operator_keys.go` - Produces multiple results - **With preferences**: `operator_to_number.go` - Configuration options - **Error handling**: `operator_error.go` - Control flow with errors - **String operations**: `operator_strings.go` - Multiple related operators ## Testing Patterns Refer to existing test files for specific patterns: - Basic expression tests in `operator_reverse_test.go` - Multi-output tests in `operator_keys_test.go` - Error handling tests in `operator_error_test.go` - Tests with `skipDoc` flag to exclude from generated documentation ## Common Patterns Refer to existing operator implementations for these patterns: - Simple transformation: see `operator_reverse.go` - Type checking: see `operator_error.go` - Working with arguments: see `operator_map.go` - Post-traversal operators: see `operator_with.go` ================================================ FILE: cmd/completion.go ================================================ package cmd import ( "os" "github.com/spf13/cobra" ) var completionCmd = &cobra.Command{ Use: "completion [bash|zsh|fish|powershell]", Aliases: []string{"shell-completion"}, Short: "Generate the autocompletion script for the specified shell", Long: `To load completions: Bash: $ source <(yq completion bash) # To load completions for each session, execute once: Linux: $ yq completion bash > /etc/bash_completion.d/yq MacOS: $ yq completion bash > /usr/local/etc/bash_completion.d/yq Zsh: # If shell completion is not already enabled in your environment you will need # to enable it. You can execute the following once: $ echo "autoload -U compinit; compinit" >> ~/.zshrc # To load completions for each session, execute once: $ yq completion zsh > "${fpath[1]}/_yq" # You will need to start a new shell for this setup to take effect. Fish: $ yq completion fish | source # To load completions for each session, execute once: $ yq completion fish > ~/.config/fish/completions/yq.fish `, DisableFlagsInUseLine: true, ValidArgs: []string{"bash", "zsh", "fish", "powershell"}, Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), RunE: func(cmd *cobra.Command, args []string) error { var err error = nil switch args[0] { case "bash": err = cmd.Root().GenBashCompletionV2(os.Stdout, true) case "zsh": err = cmd.Root().GenZshCompletion(os.Stdout) case "fish": err = cmd.Root().GenFishCompletion(os.Stdout, true) case "powershell": err = cmd.Root().GenPowerShellCompletion(os.Stdout) } return err }, } ================================================ FILE: cmd/constant.go ================================================ package cmd var unwrapScalarFlag = newUnwrapFlag() var printNodeInfo = false var unwrapScalar = false var writeInplace = false var outputToJSON = false var outputFormat = "" var inputFormat = "" var exitStatus = false var indent = 2 var noDocSeparators = false var nullInput = false var nulSepOutput = false var verbose = false var version = false var prettyPrint = false var forceColor = false var forceNoColor = false var colorsEnabled = false // can be either "" (off), "extract" or "process" var frontMatter = "" var splitFileExp = "" var splitFileExpFile = "" var completedSuccessfully = false var forceExpression = "" var expressionFile = "" ================================================ FILE: cmd/evaluate_all_command.go ================================================ package cmd import ( "errors" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" ) func createEvaluateAllCommand() *cobra.Command { var cmdEvalAll = &cobra.Command{ Use: "eval-all [expression] [yaml_file1]...", Aliases: []string{"ea"}, Short: "Loads _all_ yaml documents of _all_ yaml files and runs expression once", ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return nil, cobra.ShellCompDirectiveDefault }, Example: ` # Merge f2.yml into f1.yml (in place) yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml ## the same command and expression using shortened names: yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml # Merge all given files yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ... # Pipe from STDIN ## use '-' as a filename to pipe from STDIN cat file2.yml | yq ea '.a.b' file1.yml - file3.yml `, Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. ## Evaluate All ## This command loads _all_ yaml documents of _all_ yaml files and runs expression once Useful when you need to run an expression across several yaml documents or files (like merge). Note that it consumes more memory than eval. `, RunE: evaluateAll, } return cmdEvalAll } func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { // 0 args, read std in // 1 arg, null input, process expression // 1 arg, read file in sequence // 2+ args, [0] = expression, file the rest var err error expression, args, err := initCommand(cmd, args) if err != nil { return err } out := cmd.OutOrStdout() if writeInplace { // only use colours if its forced colorsEnabled = forceColor writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0]) out, err = writeInPlaceHandler.CreateTempFile() if err != nil { return err } // need to indirectly call the function so that completedSuccessfully is // passed when we finish execution as opposed to now defer func() { if cmdError == nil { cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) } }() } format, err := yqlib.FormatFromString(outputFormat) if err != nil { return err } decoder, err := configureDecoder(true) if err != nil { return err } printerWriter, err := configurePrinterWriter(format, out) if err != nil { return err } encoder, err := configureEncoder() if err != nil { return err } printer := yqlib.NewPrinter(encoder, printerWriter) if nulSepOutput { printer.SetNulSepOutput(true) } if frontMatter != "" { frontMatterHandler := yqlib.NewFrontMatterHandler(args[0]) err = frontMatterHandler.Split() if err != nil { return err } args[0] = frontMatterHandler.GetYamlFrontMatterFilename() if frontMatter == "process" { reader := frontMatterHandler.GetContentReader() printer.SetAppendix(reader) defer yqlib.SafelyCloseReader(reader) } defer frontMatterHandler.CleanUp() } allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator() switch len(args) { case 0: if nullInput { err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil if err == nil && exitStatus && !printer.PrintedAnything() { return errors.New("no matches found") } return err } ================================================ FILE: cmd/evaluate_all_command_test.go ================================================ package cmd import ( "bytes" "os" "path/filepath" "strings" "testing" ) func TestCreateEvaluateAllCommand(t *testing.T) { cmd := createEvaluateAllCommand() if cmd == nil { t.Fatal("createEvaluateAllCommand returned nil") } // Test basic command properties if cmd.Use != "eval-all [expression] [yaml_file1]..." { t.Errorf("Expected Use to be 'eval-all [expression] [yaml_file1]...', got %q", cmd.Use) } if cmd.Short == "" { t.Error("Expected Short description to be non-empty") } if cmd.Long == "" { t.Error("Expected Long description to be non-empty") } // Test aliases expectedAliases := []string{"ea"} if len(cmd.Aliases) != len(expectedAliases) { t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases)) } for i, expected := range expectedAliases { if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected { t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i]) } } } func TestEvaluateAll_NoArgs(t *testing.T) { // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with no arguments and no null input nullInput = false defer func() { nullInput = false }() err := evaluateAll(cmd, []string{}) // Should not error, but should print usage if err != nil { t.Errorf("evaluateAll with no args should not error, got: %v", err) } // Should have printed usage information if output.Len() == 0 { t.Error("Expected usage information to be printed") } } func TestEvaluateAll_NullInput(t *testing.T) { // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with null input nullInput = true defer func() { nullInput = false }() err := evaluateAll(cmd, []string{}) // Should not error when using null input if err != nil { t.Errorf("evaluateAll with null input should not error, got: %v", err) } } func TestEvaluateAll_WithSingleFile(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with a single file err = evaluateAll(cmd, []string{yamlFile}) // Should not error if err != nil { t.Errorf("evaluateAll with single file should not error, got: %v", err) } // Should have some output if output.Len() == 0 { t.Error("Expected output from evaluateAll with single file") } } func TestEvaluateAll_WithMultipleFiles(t *testing.T) { // Create temporary YAML files tempDir := t.TempDir() yamlFile1 := filepath.Join(tempDir, "test1.yaml") yamlContent1 := []byte("name: test1\nage: 25\n") err := os.WriteFile(yamlFile1, yamlContent1, 0600) if err != nil { t.Fatalf("Failed to create test YAML file 1: %v", err) } yamlFile2 := filepath.Join(tempDir, "test2.yaml") yamlContent2 := []byte("name: test2\nage: 30\n") err = os.WriteFile(yamlFile2, yamlContent2, 0600) if err != nil { t.Fatalf("Failed to create test YAML file 2: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with multiple files err = evaluateAll(cmd, []string{yamlFile1, yamlFile2}) // Should not error if err != nil { t.Errorf("evaluateAll with multiple files should not error, got: %v", err) } // Should have output if output.Len() == 0 { t.Error("Expected output from evaluateAll with multiple files") } } func TestEvaluateAll_WithExpression(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with expression err = evaluateAll(cmd, []string{".name", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateAll with expression should not error, got: %v", err) } // Should have output if output.Len() == 0 { t.Error("Expected output from evaluateAll with expression") } } func TestEvaluateAll_WriteInPlace(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Enable write in place originalWriteInplace := writeInplace writeInplace = true defer func() { writeInplace = originalWriteInplace }() // Test with write in place err = evaluateAll(cmd, []string{".name = \"updated\"", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateAll with write in place should not error, got: %v", err) } // Verify the file was updated updatedContent, err := os.ReadFile(yamlFile) if err != nil { t.Fatalf("Failed to read updated file: %v", err) } // Should contain the updated content if !strings.Contains(string(updatedContent), "updated") { t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent)) } } func TestEvaluateAll_ExitStatus(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Enable exit status originalExitStatus := exitStatus exitStatus = true defer func() { exitStatus = originalExitStatus }() // Test with expression that should find no matches err = evaluateAll(cmd, []string{".nonexistent", yamlFile}) // Should error when no matches found and exit status is enabled if err == nil { t.Error("Expected error when no matches found and exit status is enabled") } } func TestEvaluateAll_WithMultipleDocuments(t *testing.T) { // Create a temporary YAML file with multiple documents tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("---\nname: doc1\nage: 25\n---\nname: doc2\nage: 30\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with multiple documents err = evaluateAll(cmd, []string{".", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateAll with multiple documents should not error, got: %v", err) } // Should have output if output.Len() == 0 { t.Error("Expected output from evaluateAll with multiple documents") } } func TestEvaluateAll_NulSepOutput(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateAllCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Enable nul separator output originalNulSepOutput := nulSepOutput nulSepOutput = true defer func() { nulSepOutput = originalNulSepOutput }() // Test with nul separator output err = evaluateAll(cmd, []string{".name", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateAll with nul separator output should not error, got: %v", err) } // Should have output if output.Len() == 0 { t.Error("Expected output from evaluateAll with nul separator output") } } ================================================ FILE: cmd/evaluate_sequence_command.go ================================================ package cmd import ( "errors" "fmt" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" ) func createEvaluateSequenceCommand() *cobra.Command { var cmdEvalSequence = &cobra.Command{ Use: "eval [expression] [yaml_file1]...", Aliases: []string{"e"}, Short: "(default) Apply the expression to each document in each yaml file in sequence", ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) { if len(args) == 0 { return nil, cobra.ShellCompDirectiveNoFileComp } return nil, cobra.ShellCompDirectiveDefault }, Example: ` # Reads field under the given path for each file yq e '.a.b' f1.yml f2.yml # Prints out the file yq e sample.yaml # Pipe from STDIN ## use '-' as a filename to pipe from STDIN cat file2.yml | yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document ## Note that editing an empty file does not work. yq e -n '.a.b.c = "cat"' # Update a file in place yq e '.a.b = "cool"' -i file.yaml `, Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. ## Evaluate Sequence ## This command iterates over each yaml document from each given file, applies the expression and prints the result in sequence.`, RunE: evaluateSequence, } return cmdEvalSequence } func processExpression(expression string) string { if prettyPrint && expression == "" { return yqlib.PrettyPrintExp } else if prettyPrint { return fmt.Sprintf("%v | %v", expression, yqlib.PrettyPrintExp) } return expression } func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { // 0 args, read std in // 1 arg, null input, process expression // 1 arg, read file in sequence // 2+ args, [0] = expression, file the rest out := cmd.OutOrStdout() var err error expression, args, err := initCommand(cmd, args) if err != nil { return err } if writeInplace { // only use colours if its forced colorsEnabled = forceColor writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0]) out, err = writeInPlaceHandler.CreateTempFile() if err != nil { return err } // need to indirectly call the function so that completedSuccessfully is // passed when we finish execution as opposed to now defer func() { if cmdError == nil { cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) } }() } format, err := yqlib.FormatFromString(outputFormat) if err != nil { return err } printerWriter, err := configurePrinterWriter(format, out) if err != nil { return err } encoder, err := configureEncoder() if err != nil { return err } printer := yqlib.NewPrinter(encoder, printerWriter) if printNodeInfo { printer = yqlib.NewNodeInfoPrinter(printerWriter) } if nulSepOutput { printer.SetNulSepOutput(true) } decoder, err := configureDecoder(false) if err != nil { return err } streamEvaluator := yqlib.NewStreamEvaluator() if frontMatter != "" { yqlib.GetLogger().Debug("using front matter handler") frontMatterHandler := yqlib.NewFrontMatterHandler(args[0]) err = frontMatterHandler.Split() if err != nil { return err } args[0] = frontMatterHandler.GetYamlFrontMatterFilename() if frontMatter == "process" { reader := frontMatterHandler.GetContentReader() printer.SetAppendix(reader) defer yqlib.SafelyCloseReader(reader) } defer frontMatterHandler.CleanUp() } switch len(args) { case 0: if nullInput { err = streamEvaluator.EvaluateNew(processExpression(expression), printer) } else { cmd.Println(cmd.UsageString()) return nil } default: err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder) } completedSuccessfully = err == nil if err == nil && exitStatus && !printer.PrintedAnything() { return errors.New("no matches found") } return err } ================================================ FILE: cmd/evaluate_sequence_command_test.go ================================================ package cmd import ( "bytes" "os" "path/filepath" "strings" "testing" ) func TestCreateEvaluateSequenceCommand(t *testing.T) { cmd := createEvaluateSequenceCommand() if cmd == nil { t.Fatal("createEvaluateSequenceCommand returned nil") } // Test basic command properties if cmd.Use != "eval [expression] [yaml_file1]..." { t.Errorf("Expected Use to be 'eval [expression] [yaml_file1]...', got %q", cmd.Use) } if cmd.Short == "" { t.Error("Expected Short description to be non-empty") } if cmd.Long == "" { t.Error("Expected Long description to be non-empty") } // Test aliases expectedAliases := []string{"e"} if len(cmd.Aliases) != len(expectedAliases) { t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases)) } for i, expected := range expectedAliases { if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected { t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i]) } } } func TestProcessExpression(t *testing.T) { // Reset global variables originalPrettyPrint := prettyPrint defer func() { prettyPrint = originalPrettyPrint }() tests := []struct { name string prettyPrint bool expression string expected string }{ { name: "empty expression without pretty print", prettyPrint: false, expression: "", expected: "", }, { name: "empty expression with pretty print", prettyPrint: true, expression: "", expected: `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, }, { name: "simple expression without pretty print", prettyPrint: false, expression: ".a.b", expected: ".a.b", }, { name: "simple expression with pretty print", prettyPrint: true, expression: ".a.b", expected: `.a.b | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, }, { name: "complex expression with pretty print", prettyPrint: true, expression: ".items[] | select(.active == true)", expected: `.items[] | select(.active == true) | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { prettyPrint = tt.prettyPrint result := processExpression(tt.expression) if result != tt.expected { t.Errorf("processExpression(%q) = %q, want %q", tt.expression, result, tt.expected) } }) } } func TestEvaluateSequence_NoArgs(t *testing.T) { // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with no arguments and no null input nullInput = false defer func() { nullInput = false }() err := evaluateSequence(cmd, []string{}) // Should not error, but should print usage if err != nil { t.Errorf("evaluateSequence with no args should not error, got: %v", err) } // Should have printed usage information if output.Len() == 0 { t.Error("Expected usage information to be printed") } } func TestEvaluateSequence_NullInput(t *testing.T) { // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with null input nullInput = true defer func() { nullInput = false }() err := evaluateSequence(cmd, []string{}) // Should not error when using null input if err != nil { t.Errorf("evaluateSequence with null input should not error, got: %v", err) } } func TestEvaluateSequence_WithFile(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with a file err = evaluateSequence(cmd, []string{yamlFile}) // Should not error if err != nil { t.Errorf("evaluateSequence with file should not error, got: %v", err) } // Should have some output if output.Len() == 0 { t.Error("Expected output from evaluateSequence with file") } } func TestEvaluateSequence_WithExpressionAndFile(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Test with expression and file err = evaluateSequence(cmd, []string{".name", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateSequence with expression and file should not error, got: %v", err) } // Should have output if output.Len() == 0 { t.Error("Expected output from evaluateSequence with expression and file") } } func TestEvaluateSequence_WriteInPlace(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Enable write in place originalWriteInplace := writeInplace writeInplace = true defer func() { writeInplace = originalWriteInplace }() // Test with write in place err = evaluateSequence(cmd, []string{".name = \"updated\"", yamlFile}) // Should not error if err != nil { t.Errorf("evaluateSequence with write in place should not error, got: %v", err) } // Verify the file was updated updatedContent, err := os.ReadFile(yamlFile) if err != nil { t.Fatalf("Failed to read updated file: %v", err) } // Should contain the updated content if !strings.Contains(string(updatedContent), "updated") { t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent)) } } func TestEvaluateSequence_ExitStatus(t *testing.T) { // Create a temporary YAML file tempDir := t.TempDir() yamlFile := filepath.Join(tempDir, "test.yaml") yamlContent := []byte("name: test\nage: 25\n") err := os.WriteFile(yamlFile, yamlContent, 0600) if err != nil { t.Fatalf("Failed to create test YAML file: %v", err) } // Create a temporary command cmd := createEvaluateSequenceCommand() // Set up command to capture output var output bytes.Buffer cmd.SetOut(&output) // Enable exit status originalExitStatus := exitStatus exitStatus = true defer func() { exitStatus = originalExitStatus }() // Test with expression that should find no matches err = evaluateSequence(cmd, []string{".nonexistent", yamlFile}) // Should error when no matches found and exit status is enabled if err == nil { t.Error("Expected error when no matches found and exit status is enabled") } } ================================================ FILE: cmd/root.go ================================================ package cmd import ( "fmt" "os" "strings" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" logging "gopkg.in/op/go-logging.v1" ) type runeValue rune func newRuneVar(p *rune) *runeValue { return (*runeValue)(p) } func (r *runeValue) String() string { return string(*r) } func (r *runeValue) Set(rawVal string) error { val := strings.ReplaceAll(rawVal, "\\n", "\n") val = strings.ReplaceAll(val, "\\t", "\t") val = strings.ReplaceAll(val, "\\r", "\r") val = strings.ReplaceAll(val, "\\f", "\f") val = strings.ReplaceAll(val, "\\v", "\v") if len(val) != 1 { return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val)) } *r = runeValue(rune(val[0])) return nil } func (r *runeValue) Type() string { return "char" } func New() *cobra.Command { var rootCmd = &cobra.Command{ Use: "yq", Short: "yq is a lightweight and portable command-line data file processor.", Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`, Example: ` # yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN) # Use the '-p/--input-format' flag to specify a format type. cat file.xml | yq -p xml # read the "stuff" node from "myfile.yml" yq '.stuff' < myfile.yml # update myfile.yml in place yq -i '.stuff = "foo"' myfile.yml # print contents of sample.json as idiomatic YAML yq -P -oy sample.json `, RunE: func(cmd *cobra.Command, args []string) error { if version { cmd.Print(GetVersionDisplay()) return nil } return evaluateSequence(cmd, args) }, PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { cmd.SetOut(cmd.OutOrStdout()) level := logging.WARNING stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}` // when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled; // refer to no-color.org forceNoColor = forceNoColor || os.Getenv("NO_COLOR") != "" if verbose && forceNoColor { level = logging.DEBUG stringFormat = `[%{level:5.5s}] %{time:15:04:05} %{shortfile:-33s} %{shortfunc:-25s} %{message}` } else if verbose { level = logging.DEBUG stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}` } else if forceNoColor { stringFormat = `[%{level}] %{time:15:04:05} %{message}` } var format = logging.MustStringFormatter(stringFormat) var backend = logging.AddModuleLevel( logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format)) backend.SetLevel(level, "") logging.SetBackend(backend) yqlib.InitExpressionParser() return nil }, } rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode") rootCmd.PersistentFlags().BoolVarP(&printNodeInfo, "debug-node-info", "", false, "debug node info") rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.") err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -o=json instead") if err != nil { panic(err) } rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString())) var outputCompletions = []string{"auto"} for _, formats := range yqlib.GetAvailableOutputFormats() { outputCompletions = append(outputCompletions, formats.FormalName) } if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil { panic(err) } rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString())) var inputCompletions = []string{"auto"} for _, formats := range yqlib.GetAvailableInputFormats() { inputCompletions = append(inputCompletions, formats.FormalName) } if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil { panic(err) } rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes") if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).") if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. )") if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. )") if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values") rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix") if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix") if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)") rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "shell-key-separator", yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "separator for shell variable key paths") if err = rootCmd.RegisterFlagCompletionFunc("shell-key-separator", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)") rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output") if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit") rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.") rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colours or comments. Defaults to true for yaml") rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true" rootCmd.PersistentFlags().BoolVarP(&nulSepOutput, "nul-output", "0", false, "Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.") rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'") rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned") rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", forceNoColor, "force print with no colors") rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil { panic(err) } rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.") if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.FixMergeAnchorToSpec, "yaml-fix-merge-anchor-to-spec", "", false, "Fix merge anchor to match YAML spec. Will default to true in late 2025") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.CompactSequenceIndent, "yaml-compact-seq-indent", "c", false, "Use compact sequence indentation where '- ' is considered part of the indentation.") rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.") if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil { panic(err) } rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.") if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil { panic(err) } rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.") if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil { panic(err) } rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") rootCmd.AddCommand( createEvaluateSequenceCommand(), createEvaluateAllCommand(), completionCmd, ) return rootCmd } ================================================ FILE: cmd/root_test.go ================================================ package cmd import ( "strings" "testing" ) func TestNewRuneVar(t *testing.T) { var r rune runeVar := newRuneVar(&r) if runeVar == nil { t.Fatal("newRuneVar returned nil") } } func TestRuneValue_String(t *testing.T) { tests := []struct { name string runeVal rune expected string }{ { name: "simple character", runeVal: 'a', expected: "a", }, { name: "special character", runeVal: '\n', expected: "\n", }, { name: "unicode character", runeVal: 'ñ', expected: "ñ", }, { name: "zero rune", runeVal: 0, expected: string(rune(0)), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { runeVal := runeValue(tt.runeVal) result := runeVal.String() if result != tt.expected { t.Errorf("runeValue.String() = %q, want %q", result, tt.expected) } }) } } func TestRuneValue_Set(t *testing.T) { tests := []struct { name string input string expected rune expectError bool }{ { name: "simple character", input: "a", expected: 'a', expectError: false, }, { name: "newline escape", input: "\\n", expected: '\n', expectError: false, }, { name: "tab escape", input: "\\t", expected: '\t', expectError: false, }, { name: "carriage return escape", input: "\\r", expected: '\r', expectError: false, }, { name: "form feed escape", input: "\\f", expected: '\f', expectError: false, }, { name: "vertical tab escape", input: "\\v", expected: '\v', expectError: false, }, { name: "empty string", input: "", expected: 0, expectError: true, }, { name: "multiple characters", input: "ab", expected: 0, expectError: true, }, { name: "special character", input: "ñ", expected: 'ñ', expectError: true, // This will fail because the Set function checks len(val) != 1 }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var r rune runeVal := newRuneVar(&r) err := runeVal.Set(tt.input) if tt.expectError { if err == nil { t.Errorf("Expected error for input %q, but got none", tt.input) } } else { if err != nil { t.Errorf("Unexpected error for input %q: %v", tt.input, err) } if r != tt.expected { t.Errorf("Expected rune %q (%d), got %q (%d)", string(tt.expected), tt.expected, string(r), r) } } }) } } func TestRuneValue_Set_ErrorMessages(t *testing.T) { tests := []struct { name string input string expectedError string }{ { name: "empty string error", input: "", expectedError: "[] is not a valid character. Must be length 1 was 0", }, { name: "multiple characters error", input: "abc", expectedError: "[abc] is not a valid character. Must be length 1 was 3", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var r rune runeVal := newRuneVar(&r) err := runeVal.Set(tt.input) if err == nil { t.Errorf("Expected error for input %q, but got none", tt.input) return } if !strings.Contains(err.Error(), tt.expectedError) { t.Errorf("Expected error message to contain %q, got %q", tt.expectedError, err.Error()) } }) } } func TestRuneValue_Type(t *testing.T) { var r rune runeVal := newRuneVar(&r) result := runeVal.Type() expected := "char" if result != expected { t.Errorf("runeValue.Type() = %q, want %q", result, expected) } } func TestNew(t *testing.T) { rootCmd := New() if rootCmd == nil { t.Fatal("New() returned nil") } // Test basic command properties if rootCmd.Use != "yq" { t.Errorf("Expected Use to be 'yq', got %q", rootCmd.Use) } if rootCmd.Short == "" { t.Error("Expected Short description to be non-empty") } if rootCmd.Long == "" { t.Error("Expected Long description to be non-empty") } // Test that the command has the expected subcommands expectedCommands := []string{"eval", "eval-all", "completion"} actualCommands := make([]string, 0, len(rootCmd.Commands())) for _, cmd := range rootCmd.Commands() { actualCommands = append(actualCommands, cmd.Name()) } for _, expected := range expectedCommands { found := false for _, actual := range actualCommands { if actual == expected { found = true break } } if !found { t.Errorf("Expected command %q not found in actual commands: %v", expected, actualCommands) } } } func TestNew_FlagCompletions(t *testing.T) { rootCmd := New() // Test that flag completion functions are registered // This is a basic smoke test - we can't easily test the actual completion logic // without more complex setup flags := []string{ "output-format", "input-format", "xml-attribute-prefix", "xml-content-name", "xml-proc-inst-prefix", "xml-directive-name", "lua-prefix", "lua-suffix", "properties-separator", "indent", "front-matter", "expression", "split-exp", } for _, flagName := range flags { flag := rootCmd.PersistentFlags().Lookup(flagName) if flag == nil { t.Errorf("Expected flag %q to exist", flagName) } } } ================================================ FILE: cmd/unwrap_flag.go ================================================ package cmd import ( "strconv" "github.com/spf13/pflag" ) type boolFlag interface { pflag.Value IsExplicitlySet() bool IsSet() bool } type unwrapScalarFlagStrc struct { explicitlySet bool value bool } func newUnwrapFlag() boolFlag { return &unwrapScalarFlagStrc{value: true} } func (f *unwrapScalarFlagStrc) IsExplicitlySet() bool { return f.explicitlySet } func (f *unwrapScalarFlagStrc) IsSet() bool { return f.value } func (f *unwrapScalarFlagStrc) String() string { return strconv.FormatBool(f.value) } func (f *unwrapScalarFlagStrc) Set(value string) error { v, err := strconv.ParseBool(value) f.value = v f.explicitlySet = true return err } func (*unwrapScalarFlagStrc) Type() string { return "bool" } ================================================ FILE: cmd/utils.go ================================================ package cmd import ( "fmt" "io" "os" "strings" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" "gopkg.in/op/go-logging.v1" ) func isAutomaticOutputFormat() bool { return outputFormat == "" || outputFormat == "auto" || outputFormat == "a" } func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { cmd.SilenceUsage = true setupColors() expression, args, err := processArgs(args) if err != nil { return "", nil, err } if err := loadSplitFileExpression(); err != nil { return "", nil, err } handleBackwardsCompatibility() if err := validateCommandFlags(args); err != nil { return "", nil, err } if err := configureFormats(args); err != nil { return "", nil, err } configureUnwrapScalar() return expression, args, nil } func setupColors() { fileInfo, _ := os.Stdout.Stat() if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { colorsEnabled = true } } func loadSplitFileExpression() error { if splitFileExpFile != "" { splitExpressionBytes, err := os.ReadFile(splitFileExpFile) if err != nil { return err } splitFileExp = string(splitExpressionBytes) } return nil } func handleBackwardsCompatibility() { // backwards compatibility if outputToJSON { outputFormat = "json" } } func validateCommandFlags(args []string) error { if writeInplace && (len(args) == 0 || args[0] == "-") { return fmt.Errorf("write in place flag only applicable when giving an expression and at least one file") } if frontMatter != "" && len(args) == 0 { return fmt.Errorf("front matter flag only applicable when giving an expression and at least one file") } if writeInplace && splitFileExp != "" { return fmt.Errorf("write in place cannot be used with split file") } if nullInput && len(args) > 0 { return fmt.Errorf("cannot pass files in when using null-input flag") } return nil } func configureFormats(args []string) error { inputFilename := "" if len(args) > 0 { inputFilename = args[0] } if err := configureInputFormat(inputFilename); err != nil { return err } if err := configureOutputFormat(); err != nil { return err } yqlib.GetLogger().Debug("Using input format %v", inputFormat) yqlib.GetLogger().Debug("Using output format %v", outputFormat) return nil } func configureInputFormat(inputFilename string) error { if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" { inputFormat = yqlib.FormatStringFromFilename(inputFilename) _, err := yqlib.FormatFromString(inputFormat) if err != nil { // unknown file type, default to yaml yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat) inputFormat = "yaml" if isAutomaticOutputFormat() { outputFormat = "yaml" } } else if isAutomaticOutputFormat() { outputFormat = inputFormat } } else if isAutomaticOutputFormat() { // backwards compatibility - // before this was introduced, `yq -pcsv things.csv` // would produce *yaml* output. // outputFormat = yqlib.FormatStringFromFilename(inputFilename) if inputFilename != "-" { yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat) } outputFormat = "yaml" } return nil } func configureOutputFormat() error { outputFormatType, err := yqlib.FormatFromString(outputFormat) if err != nil { return err } if outputFormatType == yqlib.YamlFormat || outputFormatType == yqlib.PropertiesFormat { unwrapScalar = true } return nil } func configureUnwrapScalar() { if unwrapScalarFlag.IsExplicitlySet() { unwrapScalar = unwrapScalarFlag.IsSet() } } func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { format, err := yqlib.FormatFromString(inputFormat) if err != nil { return nil, err } yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether if format.DecoderFactory == nil { return nil, fmt.Errorf("no support for %s input format", inputFormat) } yqlibDecoder := format.DecoderFactory() if yqlibDecoder == nil { return nil, fmt.Errorf("no support for %s input format", inputFormat) } return yqlibDecoder, nil } func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) { var printerWriter yqlib.PrinterWriter if splitFileExp != "" { colorsEnabled = forceColor splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp) if err != nil { return nil, fmt.Errorf("bad split document expression: %w", err) } printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format) } else { printerWriter = yqlib.NewSinglePrinterWriter(out) } return printerWriter, nil } func configureEncoder() (yqlib.Encoder, error) { yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat) if err != nil { return nil, err } yqlib.ConfiguredXMLPreferences.Indent = indent yqlib.ConfiguredYamlPreferences.Indent = indent yqlib.ConfiguredKYamlPreferences.Indent = indent yqlib.ConfiguredJSONPreferences.Indent = indent yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredKYamlPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredShellVariablesPreferences.UnwrapScalar = unwrapScalar yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled yqlib.ConfiguredKYamlPreferences.ColorsEnabled = colorsEnabled yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled yqlib.ConfiguredHclPreferences.ColorsEnabled = colorsEnabled yqlib.ConfiguredTomlPreferences.ColorsEnabled = colorsEnabled yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators yqlib.ConfiguredKYamlPreferences.PrintDocSeparators = !noDocSeparators encoder := yqlibOutputFormat.EncoderFactory() if encoder == nil { return nil, fmt.Errorf("no support for %s output format", outputFormat) } return encoder, err } // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything) // and being able to call yq with the filename as a single parameter // // without this - yq detects there is stdin (thanks githubactions), // then tries to parse the filename as an expression func maybeFile(str string) bool { yqlib.GetLogger().Debugf("checking '%v' is a file", str) stat, err := os.Stat(str) // #nosec result := err == nil && !stat.IsDir() if yqlib.GetLogger().IsEnabledFor(logging.DEBUG) { if err != nil { yqlib.GetLogger().Debugf("error: %v", err) } else { yqlib.GetLogger().Debugf("error: %v, dir: %v", err, stat.IsDir()) } yqlib.GetLogger().Debugf("result: %v", result) } return result } func processStdInArgs(args []string) []string { stat, err := os.Stdin.Stat() if err != nil { yqlib.GetLogger().Debugf("error getting stdin: %v", err) } pipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0 // if we've been given a file, don't automatically // read from stdin. // this happens if there is more than one argument // or only one argument and its a file if nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) { return args } for _, arg := range args { if arg == "-" { return args } } yqlib.GetLogger().Debugf("missing '-', adding it to the end") // we're piping from stdin, but there's no '-' arg // lets add one to the end return append(args, "-") } func processArgs(originalArgs []string) (string, []string, error) { expression := forceExpression args := processStdInArgs(originalArgs) maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0]) if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") { // lets check if an expression file was given yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0]) expressionFile = args[0] args = args[1:] } if expressionFile != "" { expressionBytes, err := os.ReadFile(expressionFile) if err != nil { return "", nil, err } //replace \r\n (windows) with good ol' unix file endings. expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n") } yqlib.GetLogger().Debugf("processed args: %v", args) if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) { yqlib.GetLogger().Debug("assuming expression is '%v'", args[0]) expression = args[0] args = args[1:] } return expression, args, nil } ================================================ FILE: cmd/utils_test.go ================================================ package cmd import ( "fmt" "os" "strings" "testing" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" ) func TestIsAutomaticOutputFormat(t *testing.T) { tests := []struct { name string format string expected bool }{ {"empty format", "", true}, {"auto format", "auto", true}, {"short auto format", "a", true}, {"json format", "json", false}, {"yaml format", "yaml", false}, {"xml format", "xml", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original value originalFormat := outputFormat defer func() { outputFormat = originalFormat }() outputFormat = tt.format result := isAutomaticOutputFormat() if result != tt.expected { t.Errorf("isAutomaticOutputFormat() = %v, want %v", result, tt.expected) } }) } } func TestMaybeFile(t *testing.T) { // Create a temporary file for testing tempFile, err := os.CreateTemp("", "test") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Create a temporary directory for testing tempDir, err := os.MkdirTemp("", "test") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } defer os.RemoveAll(tempDir) tests := []struct { name string path string expected bool }{ {"existing file", tempFile.Name(), true}, {"existing directory", tempDir, false}, {"non-existent path", "/path/that/does/not/exist", false}, {"empty string", "", false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := maybeFile(tt.path) if result != tt.expected { t.Errorf("maybeFile(%q) = %v, want %v", tt.path, result, tt.expected) } }) } } func TestProcessArgs(t *testing.T) { // Create a temporary file for testing tempFile, err := os.CreateTemp("", "test") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Create a temporary .yq file for testing tempYqFile, err := os.Create("test.yq") if err != nil { t.Fatalf("Failed to create temp yq file: %v", err) } defer os.Remove(tempYqFile.Name()) if _, err = tempYqFile.WriteString(".a.b"); err != nil { t.Fatalf("Failed to write to temp yq file: %v", err) } tempYqFile.Close() tests := []struct { name string args []string forceExpression string expressionFile string expectedExpr string expectedArgs []string expectError bool }{ { name: "empty args", args: []string{}, forceExpression: "", expressionFile: "", expectedExpr: "", expectedArgs: []string{}, expectError: false, }, { name: "force expression", args: []string{"file1"}, forceExpression: ".a.b", expressionFile: "", expectedExpr: ".a.b", expectedArgs: []string{"file1"}, expectError: false, }, { name: "expression as first arg", args: []string{".a.b", "file1"}, forceExpression: "", expressionFile: "", expectedExpr: ".a.b", expectedArgs: []string{"file1"}, expectError: false, }, { name: "file as first arg", args: []string{tempFile.Name()}, forceExpression: "", expressionFile: "", expectedExpr: "", expectedArgs: []string{tempFile.Name()}, expectError: false, }, { name: "yq file as first arg", args: []string{tempYqFile.Name(), "things"}, forceExpression: "", expressionFile: "", expectedExpr: ".a.b", expectedArgs: []string{"things"}, expectError: false, }, { name: "dash as first arg", args: []string{"-"}, forceExpression: "", expressionFile: "", expectedExpr: "", expectedArgs: []string{"-"}, expectError: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalForceExpression := forceExpression originalExpressionFile := expressionFile defer func() { forceExpression = originalForceExpression expressionFile = originalExpressionFile }() forceExpression = tt.forceExpression expressionFile = tt.expressionFile expr, args, err := processArgs(tt.args) if tt.expectError { if err == nil { t.Errorf("processArgs() expected error but got none") } return } if err != nil { t.Errorf("processArgs() unexpected error: %v", err) return } if expr != tt.expectedExpr { t.Errorf("processArgs() expression = %v, want %v", expr, tt.expectedExpr) } if !stringsEqual(args, tt.expectedArgs) { t.Errorf("processArgs() args = %v, want %v", args, tt.expectedArgs) } }) } } func TestConfigureDecoder(t *testing.T) { tests := []struct { name string inputFormat string evaluateTogether bool expectError bool expectType string }{ { name: "yaml format", inputFormat: "yaml", evaluateTogether: false, expectError: false, expectType: "yamlDecoder", }, { name: "json format", inputFormat: "json", evaluateTogether: true, expectError: false, expectType: "jsonDecoder", }, { name: "xml format", inputFormat: "xml", evaluateTogether: false, expectError: false, expectType: "xmlDecoder", }, { name: "invalid format", inputFormat: "invalid", evaluateTogether: false, expectError: true, expectType: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original value originalInputFormat := inputFormat defer func() { inputFormat = originalInputFormat }() inputFormat = tt.inputFormat decoder, err := configureDecoder(tt.evaluateTogether) if tt.expectError { if err == nil { t.Errorf("configureDecoder() expected error but got none") } if decoder != nil { t.Errorf("configureDecoder() expected nil decoder but got %v", decoder) } return } if err != nil { t.Errorf("configureDecoder() unexpected error: %v", err) return } if decoder == nil { t.Errorf("configureDecoder() expected decoder but got nil") return } typeStr := fmt.Sprintf("%T", decoder) if !strings.Contains(typeStr, tt.expectType) { t.Errorf("configureDecoder() expected type to contain %q but got %q", tt.expectType, typeStr) } }) } } func TestConfigurePrinterWriter(t *testing.T) { yqlib.InitExpressionParser() tests := []struct { name string splitFileExp string format *yqlib.Format forceColor bool expectError bool expectMulti bool expectColorsEnabled bool }{ { name: "single printer writer", splitFileExp: "", format: &yqlib.Format{}, forceColor: false, expectError: false, expectMulti: false, expectColorsEnabled: false, }, { name: "multi printer writer with valid expression", splitFileExp: ".a.b", format: &yqlib.Format{}, forceColor: true, expectError: false, expectMulti: true, expectColorsEnabled: true, }, { name: "multi printer writer with invalid expression", splitFileExp: "[invalid", format: &yqlib.Format{}, forceColor: false, expectError: true, expectMulti: false, expectColorsEnabled: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalSplitFileExp := splitFileExp originalForceColor := forceColor originalColorsEnabled := colorsEnabled defer func() { splitFileExp = originalSplitFileExp forceColor = originalForceColor colorsEnabled = originalColorsEnabled }() splitFileExp = tt.splitFileExp forceColor = tt.forceColor colorsEnabled = false // Reset to test the setting writer, err := configurePrinterWriter(tt.format, os.Stdout) if tt.expectError { if err == nil { t.Errorf("configurePrinterWriter() expected error but got none") } if writer != nil { t.Errorf("configurePrinterWriter() expected nil writer but got %v", writer) } return } if err != nil { t.Errorf("configurePrinterWriter() unexpected error: %v", err) return } if writer == nil { t.Errorf("configurePrinterWriter() expected writer but got nil") return } // Explicitly check colorsEnabled if colorsEnabled != tt.expectColorsEnabled { t.Errorf("configurePrinterWriter() colorsEnabled = %v, want %v", colorsEnabled, tt.expectColorsEnabled) } // Check the type of the returned writer writerType := fmt.Sprintf("%T", writer) if tt.expectMulti { if !strings.Contains(writerType, "multiPrintWriter") { t.Errorf("configurePrinterWriter() expected multiPrintWriter but got %s", writerType) } } else { if !strings.Contains(writerType, "singlePrinterWriter") { t.Errorf("configurePrinterWriter() expected singlePrinterWriter but got %s", writerType) } } }) } } func TestConfigureEncoder(t *testing.T) { tests := []struct { name string outputFormat string expectError bool expectType string }{ { name: "yaml format", outputFormat: "yaml", expectError: false, expectType: "yamlEncoder", }, { name: "json format", outputFormat: "json", expectError: false, expectType: "jsonEncoder", }, { name: "xml format", outputFormat: "xml", expectError: false, expectType: "xmlEncoder", }, { name: "properties format", outputFormat: "properties", expectError: false, expectType: "propertiesEncoder", }, { name: "invalid format", outputFormat: "invalid", expectError: true, expectType: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalOutputFormat := outputFormat originalIndent := indent originalUnwrapScalar := unwrapScalar originalColorsEnabled := colorsEnabled originalNoDocSeparators := noDocSeparators defer func() { outputFormat = originalOutputFormat indent = originalIndent unwrapScalar = originalUnwrapScalar colorsEnabled = originalColorsEnabled noDocSeparators = originalNoDocSeparators }() outputFormat = tt.outputFormat indent = 2 unwrapScalar = false colorsEnabled = false noDocSeparators = false encoder, err := configureEncoder() if tt.expectError { if err == nil { t.Errorf("configureEncoder() expected error but got none") } if encoder != nil { t.Errorf("configureEncoder() expected nil encoder but got %v", encoder) } return } if err != nil { t.Errorf("configureEncoder() unexpected error: %v", err) return } if encoder == nil { t.Errorf("configureEncoder() expected encoder but got nil") return } typeStr := fmt.Sprintf("%T", encoder) if !strings.Contains(typeStr, tt.expectType) { t.Errorf("configureEncoder() expected type to contain %q but got %q", tt.expectType, typeStr) } }) } } func TestInitCommand(t *testing.T) { // Create a temporary file for testing tempFile, err := os.CreateTemp("", "test") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Create a temporary split file tempSplitFile, err := os.CreateTemp("", "split") if err != nil { t.Fatalf("Failed to create temp split file: %v", err) } defer os.Remove(tempSplitFile.Name()) if _, err = tempSplitFile.WriteString(".a.b"); err != nil { t.Fatalf("Failed to write to temp split file: %v", err) } tempSplitFile.Close() tests := []struct { name string args []string writeInplace bool frontMatter string nullInput bool splitFileExpFile string splitFileExp string outputToJSON bool expectError bool errorContains string expectExpr string expectArgs []string }{ { name: "basic command", args: []string{tempFile.Name()}, writeInplace: false, frontMatter: "", nullInput: false, expectError: false, expectExpr: "", expectArgs: []string{tempFile.Name()}, }, { name: "write inplace with no args", args: []string{}, writeInplace: true, frontMatter: "", nullInput: false, expectError: true, errorContains: "write in place flag only applicable when giving an expression and at least one file", }, { name: "split file expression from file", args: []string{tempFile.Name()}, writeInplace: false, frontMatter: "", nullInput: false, splitFileExpFile: tempSplitFile.Name(), expectError: false, expectExpr: "", expectArgs: []string{tempFile.Name()}, }, { name: "output to JSON", args: []string{tempFile.Name()}, writeInplace: false, frontMatter: "", nullInput: false, outputToJSON: true, expectError: false, expectExpr: "", expectArgs: []string{tempFile.Name()}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalWriteInplace := writeInplace originalFrontMatter := frontMatter originalNullInput := nullInput originalSplitFileExpFile := splitFileExpFile originalSplitFileExp := splitFileExp originalOutputToJSON := outputToJSON originalInputFormat := inputFormat originalOutputFormat := outputFormat originalForceColor := forceColor originalForceNoColor := forceNoColor originalColorsEnabled := colorsEnabled defer func() { writeInplace = originalWriteInplace frontMatter = originalFrontMatter nullInput = originalNullInput splitFileExpFile = originalSplitFileExpFile splitFileExp = originalSplitFileExp outputToJSON = originalOutputToJSON inputFormat = originalInputFormat outputFormat = originalOutputFormat forceColor = originalForceColor forceNoColor = originalForceNoColor colorsEnabled = originalColorsEnabled }() writeInplace = tt.writeInplace frontMatter = tt.frontMatter nullInput = tt.nullInput splitFileExpFile = tt.splitFileExpFile splitFileExp = tt.splitFileExp outputToJSON = tt.outputToJSON inputFormat = "auto" outputFormat = "auto" forceColor = false forceNoColor = false colorsEnabled = false cmd := &cobra.Command{} expr, args, err := initCommand(cmd, tt.args) if tt.expectError { if err == nil { t.Errorf("initCommand() expected error but got none") return } if tt.errorContains != "" && !strings.Contains(err.Error(), tt.errorContains) { t.Errorf("initCommand() error '%v' does not contain '%v'", err.Error(), tt.errorContains) } return } if err != nil { t.Errorf("initCommand() unexpected error: %v", err) return } if expr != tt.expectExpr { t.Errorf("initCommand() expr = %v, want %v", expr, tt.expectExpr) } if !stringsEqual(args, tt.expectArgs) { t.Errorf("initCommand() args = %v, want %v", args, tt.expectArgs) } }) } } func TestProcessArgsWithExpressionFile(t *testing.T) { // Create a temporary .yq file with Windows line endings tempYqFile, err := os.CreateTemp("", "test.yq") if err != nil { t.Fatalf("Failed to create temp yq file: %v", err) } defer os.Remove(tempYqFile.Name()) if _, err = tempYqFile.WriteString(".a.b\r\n.c.d"); err != nil { t.Fatalf("Failed to write to temp yq file: %v", err) } tempYqFile.Close() // Save original values originalExpressionFile := expressionFile defer func() { expressionFile = originalExpressionFile }() expressionFile = tempYqFile.Name() expr, args, err := processArgs([]string{"file1"}) if err != nil { t.Errorf("processArgs() unexpected error: %v", err) return } expectedExpr := ".a.b\n.c.d" // Should convert \r\n to \n if expr != expectedExpr { t.Errorf("processArgs() expression = %v, want %v", expr, expectedExpr) } expectedArgs := []string{"file1"} if !stringsEqual(args, expectedArgs) { t.Errorf("processArgs() args = %v, want %v", args, expectedArgs) } } func TestProcessArgsWithNonExistentExpressionFile(t *testing.T) { // Save original values originalExpressionFile := expressionFile defer func() { expressionFile = originalExpressionFile }() expressionFile = "/path/that/does/not/exist" expr, args, err := processArgs([]string{"file1"}) if err == nil { t.Errorf("processArgs() expected error but got none") } if expr != "" { t.Errorf("processArgs() expected empty expression but got %v", expr) } if args != nil { t.Errorf("processArgs() expected nil args but got %v", args) } } func TestInitCommandWithInvalidOutputFormat(t *testing.T) { // Create a temporary file for testing tempFile, err := os.CreateTemp("", "test") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Save original values originalInputFormat := inputFormat originalOutputFormat := outputFormat defer func() { inputFormat = originalInputFormat outputFormat = originalOutputFormat }() inputFormat = "auto" outputFormat = "invalid" cmd := &cobra.Command{} expr, args, err := initCommand(cmd, []string{tempFile.Name()}) if err == nil { t.Errorf("initCommand() expected error but got none") } if expr != "" { t.Errorf("initCommand() expected empty expression but got %v", expr) } if args != nil { t.Errorf("initCommand() expected nil args but got %v", args) } } func TestInitCommandWithUnknownInputFormat(t *testing.T) { // Create a temporary file with unknown extension tempFile, err := os.CreateTemp("", "test.unknown") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Save original values originalInputFormat := inputFormat originalOutputFormat := outputFormat defer func() { inputFormat = originalInputFormat outputFormat = originalOutputFormat }() inputFormat = "auto" outputFormat = "auto" cmd := &cobra.Command{} expr, args, err := initCommand(cmd, []string{tempFile.Name()}) if err != nil { t.Errorf("initCommand() unexpected error: %v", err) return } // expr can be empty when no expression is provided _ = expr if args == nil { t.Errorf("initCommand() expected non-nil args") } } func TestConfigurePrinterWriterWithInvalidSplitExpression(t *testing.T) { // Save original value originalSplitFileExp := splitFileExp defer func() { splitFileExp = originalSplitFileExp }() splitFileExp = "[invalid expression" writer, err := configurePrinterWriter(&yqlib.Format{}, os.Stdout) if err == nil { t.Errorf("configurePrinterWriter() expected error but got none") } if writer != nil { t.Errorf("configurePrinterWriter() expected nil writer but got %v", writer) } if err != nil && !strings.Contains(err.Error(), "bad split document expression") { t.Errorf("configurePrinterWriter() error '%v' does not contain expected message", err.Error()) } } func TestMaybeFileWithDirectory(t *testing.T) { // Create a temporary directory tempDir, err := os.MkdirTemp("", "test") if err != nil { t.Fatalf("Failed to create temp dir: %v", err) } defer os.RemoveAll(tempDir) result := maybeFile(tempDir) if result { t.Errorf("maybeFile(%q) = %v, want false", tempDir, result) } } func TestProcessStdInArgsWithDash(t *testing.T) { args := []string{"-", "file1"} result := processStdInArgs(args) if !stringsEqual(result, args) { t.Errorf("processStdInArgs() = %v, want %v", result, args) } } func TestProcessArgsWithYqFileExtension(t *testing.T) { tempYqFile, err := os.Create("test.yq") if err != nil { t.Fatalf("Failed to create temp yq file: %v", err) } defer os.Remove(tempYqFile.Name()) if _, err = tempYqFile.WriteString(".a.b"); err != nil { t.Fatalf("Failed to write to temp yq file: %v", err) } tempYqFile.Close() // Save original values originalExpressionFile := expressionFile originalForceExpression := forceExpression defer func() { expressionFile = originalExpressionFile forceExpression = originalForceExpression }() // Reset expressionFile to empty to test the auto-detection expressionFile = "" forceExpression = "" // Debug: check the conditions manually t.Logf("expressionFile: %q", expressionFile) t.Logf("forceExpression: %q", forceExpression) t.Logf("tempYqFile.Name(): %q", tempYqFile.Name()) t.Logf("strings.HasSuffix(tempYqFile.Name(), '.yq'): %v", strings.HasSuffix(tempYqFile.Name(), ".yq")) t.Logf("maybeFile(tempYqFile.Name()): %v", maybeFile(tempYqFile.Name())) // Test with only the yq file as argument (should be treated as expression file) expr, args, err := processArgs([]string{tempYqFile.Name()}) if err != nil { t.Errorf("processArgs() unexpected error: %v", err) return } if expr != ".a.b" { t.Errorf("processArgs() expression = %v, want .a.b", expr) } expectedArgs := []string{} if !stringsEqual(args, expectedArgs) { t.Errorf("processArgs() args = %v, want %v", args, expectedArgs) } } func TestConfigureEncoderWithYamlFormat(t *testing.T) { // Save original values originalOutputFormat := outputFormat originalIndent := indent originalUnwrapScalar := unwrapScalar originalColorsEnabled := colorsEnabled originalNoDocSeparators := noDocSeparators defer func() { outputFormat = originalOutputFormat indent = originalIndent unwrapScalar = originalUnwrapScalar colorsEnabled = originalColorsEnabled noDocSeparators = originalNoDocSeparators }() outputFormat = "yaml" indent = 4 unwrapScalar = true colorsEnabled = true noDocSeparators = true encoder, err := configureEncoder() if err != nil { t.Errorf("configureEncoder() unexpected error: %v", err) return } if encoder == nil { t.Errorf("configureEncoder() expected encoder but got nil") } } func TestConfigureEncoderWithPropertiesFormat(t *testing.T) { // Save original values originalOutputFormat := outputFormat originalIndent := indent originalUnwrapScalar := unwrapScalar originalColorsEnabled := colorsEnabled originalNoDocSeparators := noDocSeparators defer func() { outputFormat = originalOutputFormat indent = originalIndent unwrapScalar = originalUnwrapScalar colorsEnabled = originalColorsEnabled noDocSeparators = originalNoDocSeparators }() outputFormat = "properties" indent = 2 unwrapScalar = false colorsEnabled = false noDocSeparators = false encoder, err := configureEncoder() if err != nil { t.Errorf("configureEncoder() unexpected error: %v", err) return } if encoder == nil { t.Errorf("configureEncoder() expected encoder but got nil") } } // Mock boolFlag for testing type mockBoolFlag struct { explicitlySet bool value bool } func (f *mockBoolFlag) IsExplicitlySet() bool { return f.explicitlySet } func (f *mockBoolFlag) IsSet() bool { return f.value } func (f *mockBoolFlag) String() string { return "mock" } func (f *mockBoolFlag) Set(_ string) error { return nil } func (f *mockBoolFlag) Type() string { return "bool" } // Helper function to compare string slices func stringsEqual(a, b []string) bool { if len(a) != len(b) { return false } for i := range a { if a[i] != b[i] { return false } } return true } func TestSetupColors(t *testing.T) { tests := []struct { name string forceColor bool forceNoColor bool expectColors bool }{ { name: "force colour enabled", forceColor: true, forceNoColor: false, expectColors: true, }, { name: "force no colour enabled", forceColor: false, forceNoColor: true, expectColors: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalForceColor := forceColor originalForceNoColor := forceNoColor originalColorsEnabled := colorsEnabled defer func() { forceColor = originalForceColor forceNoColor = originalForceNoColor colorsEnabled = originalColorsEnabled }() forceColor = tt.forceColor forceNoColor = tt.forceNoColor colorsEnabled = false // Reset to test the setting setupColors() if colorsEnabled != tt.expectColors { t.Errorf("setupColors() colorsEnabled = %v, want %v", colorsEnabled, tt.expectColors) } }) } } func TestLoadSplitFileExpression(t *testing.T) { // Create a temporary file with expression content tempFile, err := os.CreateTemp("", "split") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) if _, err = tempFile.WriteString(".a.b"); err != nil { t.Fatalf("Failed to write to temp file: %v", err) } tempFile.Close() tests := []struct { name string splitFileExpFile string expectError bool expectContent string }{ { name: "load from file", splitFileExpFile: tempFile.Name(), expectError: false, expectContent: ".a.b", }, { name: "no file specified", splitFileExpFile: "", expectError: false, expectContent: "", }, { name: "non-existent file", splitFileExpFile: "/path/that/does/not/exist", expectError: true, expectContent: "", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original value originalSplitFileExpFile := splitFileExpFile originalSplitFileExp := splitFileExp defer func() { splitFileExpFile = originalSplitFileExpFile splitFileExp = originalSplitFileExp }() splitFileExpFile = tt.splitFileExpFile splitFileExp = "" err := loadSplitFileExpression() if tt.expectError { if err == nil { t.Errorf("loadSplitFileExpression() expected error but got none") } return } if err != nil { t.Errorf("loadSplitFileExpression() unexpected error: %v", err) return } if splitFileExp != tt.expectContent { t.Errorf("loadSplitFileExpression() splitFileExp = %v, want %v", splitFileExp, tt.expectContent) } }) } } func TestHandleBackwardsCompatibility(t *testing.T) { tests := []struct { name string outputToJSON bool initialFormat string expectFormat string }{ { name: "outputToJSON true", outputToJSON: true, initialFormat: "yaml", expectFormat: "json", }, { name: "outputToJSON false", outputToJSON: false, initialFormat: "yaml", expectFormat: "yaml", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original value originalOutputToJSON := outputToJSON originalOutputFormat := outputFormat defer func() { outputToJSON = originalOutputToJSON outputFormat = originalOutputFormat }() outputToJSON = tt.outputToJSON outputFormat = tt.initialFormat handleBackwardsCompatibility() if outputFormat != tt.expectFormat { t.Errorf("handleBackwardsCompatibility() outputFormat = %v, want %v", outputFormat, tt.expectFormat) } }) } } func TestValidateCommandFlags(t *testing.T) { tests := []struct { name string args []string writeInplace bool frontMatter string splitFileExp string nullInput bool expectError bool errorContains string }{ { name: "valid flags", args: []string{"file.yaml"}, writeInplace: false, frontMatter: "", splitFileExp: "", nullInput: false, expectError: false, }, { name: "write inplace with no args", args: []string{}, writeInplace: true, frontMatter: "", splitFileExp: "", nullInput: false, expectError: true, errorContains: "write in place flag only applicable when giving an expression and at least one file", }, { name: "write inplace with dash", args: []string{"-"}, writeInplace: true, frontMatter: "", splitFileExp: "", nullInput: false, expectError: true, errorContains: "write in place flag only applicable when giving an expression and at least one file", }, { name: "front matter with no args", args: []string{}, writeInplace: false, frontMatter: "extract", splitFileExp: "", nullInput: false, expectError: true, errorContains: "front matter flag only applicable when giving an expression and at least one file", }, { name: "write inplace with split file", args: []string{"file.yaml"}, writeInplace: true, frontMatter: "", splitFileExp: ".a.b", nullInput: false, expectError: true, errorContains: "write in place cannot be used with split file", }, { name: "null input with args", args: []string{"file.yaml"}, writeInplace: false, frontMatter: "", splitFileExp: "", nullInput: true, expectError: true, errorContains: "cannot pass files in when using null-input flag", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalWriteInplace := writeInplace originalFrontMatter := frontMatter originalSplitFileExp := splitFileExp originalNullInput := nullInput defer func() { writeInplace = originalWriteInplace frontMatter = originalFrontMatter splitFileExp = originalSplitFileExp nullInput = originalNullInput }() writeInplace = tt.writeInplace frontMatter = tt.frontMatter splitFileExp = tt.splitFileExp nullInput = tt.nullInput err := validateCommandFlags(tt.args) if tt.expectError { if err == nil { t.Errorf("validateCommandFlags() expected error but got none") return } if tt.errorContains != "" && !strings.Contains(err.Error(), tt.errorContains) { t.Errorf("validateCommandFlags() error '%v' does not contain '%v'", err.Error(), tt.errorContains) } return } if err != nil { t.Errorf("validateCommandFlags() unexpected error: %v", err) } }) } } func TestConfigureFormats(t *testing.T) { tests := []struct { name string args []string inputFormat string outputFormat string expectError bool }{ { name: "valid formats", args: []string{"file.yaml"}, inputFormat: "auto", outputFormat: "auto", expectError: false, }, { name: "invalid output format", args: []string{"file.yaml"}, inputFormat: "auto", outputFormat: "invalid", expectError: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalInputFormat := inputFormat originalOutputFormat := outputFormat defer func() { inputFormat = originalInputFormat outputFormat = originalOutputFormat }() inputFormat = tt.inputFormat outputFormat = tt.outputFormat err := configureFormats(tt.args) if tt.expectError { if err == nil { t.Errorf("configureFormats() expected error but got none") } return } if err != nil { t.Errorf("configureFormats() unexpected error: %v", err) } }) } } func TestConfigureInputFormat(t *testing.T) { tests := []struct { name string inputFilename string inputFormat string outputFormat string expectInput string expectOutput string }{ { name: "auto format with yaml file", inputFilename: "file.yaml", inputFormat: "auto", outputFormat: "auto", expectInput: "yaml", expectOutput: "yaml", }, { name: "auto format with json file", inputFilename: "file.json", inputFormat: "auto", outputFormat: "auto", expectInput: "json", expectOutput: "json", }, { name: "auto format with unknown file", inputFilename: "file.unknown", inputFormat: "auto", outputFormat: "auto", expectInput: "yaml", expectOutput: "yaml", }, { name: "explicit format", inputFilename: "file.yaml", inputFormat: "json", outputFormat: "auto", expectInput: "json", expectOutput: "yaml", // backwards compatibility }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalInputFormat := inputFormat originalOutputFormat := outputFormat defer func() { inputFormat = originalInputFormat outputFormat = originalOutputFormat }() inputFormat = tt.inputFormat outputFormat = tt.outputFormat err := configureInputFormat(tt.inputFilename) if err != nil { t.Errorf("configureInputFormat() unexpected error: %v", err) return } if inputFormat != tt.expectInput { t.Errorf("configureInputFormat() inputFormat = %v, want %v", inputFormat, tt.expectInput) } if outputFormat != tt.expectOutput { t.Errorf("configureInputFormat() outputFormat = %v, want %v", outputFormat, tt.expectOutput) } }) } } func TestConfigureOutputFormat(t *testing.T) { tests := []struct { name string outputFormat string expectError bool expectUnwrap bool }{ { name: "yaml format", outputFormat: "yaml", expectError: false, expectUnwrap: true, }, { name: "properties format", outputFormat: "properties", expectError: false, expectUnwrap: true, }, { name: "json format", outputFormat: "json", expectError: false, expectUnwrap: false, }, { name: "invalid format", outputFormat: "invalid", expectError: true, expectUnwrap: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original values originalOutputFormat := outputFormat originalUnwrapScalar := unwrapScalar defer func() { outputFormat = originalOutputFormat unwrapScalar = originalUnwrapScalar }() outputFormat = tt.outputFormat unwrapScalar = false // Reset to test the setting err := configureOutputFormat() if tt.expectError { if err == nil { t.Errorf("configureOutputFormat() expected error but got none") } return } if err != nil { t.Errorf("configureOutputFormat() unexpected error: %v", err) return } if unwrapScalar != tt.expectUnwrap { t.Errorf("configureOutputFormat() unwrapScalar = %v, want %v", unwrapScalar, tt.expectUnwrap) } }) } } func TestConfigureUnwrapScalar(t *testing.T) { tests := []struct { name string explicitlySet bool flagValue bool initialUnwrap bool expectUnwrap bool }{ { name: "flag not explicitly set", explicitlySet: false, flagValue: true, initialUnwrap: true, expectUnwrap: true, // Should remain unchanged }, { name: "flag explicitly set to true", explicitlySet: true, flagValue: true, initialUnwrap: false, expectUnwrap: true, }, { name: "flag explicitly set to false", explicitlySet: true, flagValue: false, initialUnwrap: true, expectUnwrap: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Save original value originalUnwrapScalar := unwrapScalar originalUnwrapScalarFlag := unwrapScalarFlag defer func() { unwrapScalar = originalUnwrapScalar unwrapScalarFlag = originalUnwrapScalarFlag }() unwrapScalar = tt.initialUnwrap unwrapScalarFlag = &mockBoolFlag{ explicitlySet: tt.explicitlySet, value: tt.flagValue, } configureUnwrapScalar() if unwrapScalar != tt.expectUnwrap { t.Errorf("configureUnwrapScalar() unwrapScalar = %v, want %v", unwrapScalar, tt.expectUnwrap) } }) } } ================================================ FILE: cmd/version.go ================================================ package cmd import ( "fmt" "strings" ) // The git commit that was compiled. This will be filled in by the compiler. var ( GitCommit string GitDescribe string // Version is main version number that is being run at the moment. Version = "v4.52.4" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release // such as "dev" (in development), "beta", "rc1", etc. VersionPrerelease = "" ) // ProductName is the name of the product const ProductName = "yq" // GetVersionDisplay composes the parts of the version in a way that's suitable // for displaying to humans. func GetVersionDisplay() string { return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion()) } func getHumanVersion() string { version := Version if GitDescribe != "" { version = GitDescribe } release := VersionPrerelease if release != "" { if !strings.Contains(version, release) { version += fmt.Sprintf("-%s", release) } if GitCommit != "" { version += fmt.Sprintf(" (%s)", GitCommit) } } // Strip off any single quotes added by the git information. return strings.ReplaceAll(version, "'", "") } ================================================ FILE: cmd/version_test.go ================================================ package cmd import ( "strings" "testing" ) func TestGetVersionDisplay(t *testing.T) { var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version if VersionPrerelease != "" { expectedVersion = expectedVersion + "-" + VersionPrerelease } expectedVersion = expectedVersion + "\n" tests := []struct { name string want string }{ { name: "Display Version", want: expectedVersion, }, } for _, tt := range tests { if got := GetVersionDisplay(); got != tt.want { t.Errorf("%q. GetVersionDisplay() = %v, want %v", tt.name, got, tt.want) } } } func Test_getHumanVersion(t *testing.T) { // Save original values origGitDescribe := GitDescribe origGitCommit := GitCommit origVersionPrerelease := VersionPrerelease // Restore after test defer func() { GitDescribe = origGitDescribe GitCommit = origGitCommit VersionPrerelease = origVersionPrerelease }() GitDescribe = "e42813d" GitCommit = "e42813d+CHANGES" var wanted string if VersionPrerelease == "" { wanted = GitDescribe } else { wanted = "e42813d-" + VersionPrerelease + " (e42813d+CHANGES)" } tests := []struct { name string want string }{ { name: "Git Variables defined", want: wanted, }, } for _, tt := range tests { if got := getHumanVersion(); got != tt.want { t.Errorf("%q. getHumanVersion() = %v, want %v", tt.name, got, tt.want) } } } func Test_getHumanVersion_NoGitDescribe(t *testing.T) { // Save original values origGitDescribe := GitDescribe origGitCommit := GitCommit origVersionPrerelease := VersionPrerelease // Restore after test defer func() { GitDescribe = origGitDescribe GitCommit = origGitCommit VersionPrerelease = origVersionPrerelease }() GitDescribe = "" GitCommit = "" VersionPrerelease = "" got := getHumanVersion() if got != Version { t.Errorf("getHumanVersion() = %v, want %v", got, Version) } } func Test_getHumanVersion_WithPrerelease(t *testing.T) { // Save original values origGitDescribe := GitDescribe origGitCommit := GitCommit origVersionPrerelease := VersionPrerelease // Restore after test defer func() { GitDescribe = origGitDescribe GitCommit = origGitCommit VersionPrerelease = origVersionPrerelease }() GitDescribe = "" GitCommit = "abc123" VersionPrerelease = "beta" got := getHumanVersion() expected := Version + "-beta (abc123)" if got != expected { t.Errorf("getHumanVersion() = %v, want %v", got, expected) } } func Test_getHumanVersion_PrereleaseInVersion(t *testing.T) { // Save original values origGitDescribe := GitDescribe origGitCommit := GitCommit origVersionPrerelease := VersionPrerelease // Restore after test defer func() { GitDescribe = origGitDescribe GitCommit = origGitCommit VersionPrerelease = origVersionPrerelease }() GitDescribe = "v1.2.3-rc1" GitCommit = "xyz789" VersionPrerelease = "rc1" got := getHumanVersion() // Should not duplicate "rc1" since it's already in GitDescribe expected := "v1.2.3-rc1 (xyz789)" if got != expected { t.Errorf("getHumanVersion() = %v, want %v", got, expected) } } func Test_getHumanVersion_StripSingleQuotes(t *testing.T) { // Save original values origGitDescribe := GitDescribe origGitCommit := GitCommit origVersionPrerelease := VersionPrerelease // Restore after test defer func() { GitDescribe = origGitDescribe GitCommit = origGitCommit VersionPrerelease = origVersionPrerelease }() GitDescribe = "'v1.2.3'" GitCommit = "'commit123'" VersionPrerelease = "" got := getHumanVersion() // Should strip single quotes if strings.Contains(got, "'") { t.Errorf("getHumanVersion() = %v, should not contain single quotes", got) } expected := "v1.2.3" if got != expected { t.Errorf("getHumanVersion() = %v, want %v", got, expected) } } func TestProductName(t *testing.T) { if ProductName != "yq" { t.Errorf("ProductName = %v, want yq", ProductName) } } func TestVersionIsSet(t *testing.T) { if Version == "" { t.Error("Version should not be empty") } if !strings.HasPrefix(Version, "v") { t.Errorf("Version %v should start with 'v'", Version) } } ================================================ FILE: cspell.config.yaml ================================================ --- $schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json version: '0.2' language: en-GB dictionaryDefinitions: - name: project-words path: './project-words.txt' addWords: true dictionaries: - project-words ignorePaths: - 'vendor' - 'bin' - '/project-words.txt' ================================================ FILE: examples/array.yaml ================================================ - [cat, dog, frog, cow] - [apple, banana, grape, mango] ================================================ FILE: examples/bad.yaml ================================================ b: d: be gone c: 2 e: - name: Billy Bob # comment over here --- [123123 ================================================ FILE: examples/base64.txt ================================================ bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u ================================================ FILE: examples/data.lua ================================================ return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; ================================================ FILE: examples/data1-no-comments.yaml ================================================ a: simple b: [1, 2] c: test: 1 ================================================ FILE: examples/data1.yaml ================================================ a: apple ================================================ FILE: examples/data2.yaml ================================================ # -------------------------------------------------- # It's a test with comment # -------------------------------------------------- groups: - name: d ================================================ FILE: examples/data3.yaml ================================================ a: "simple" # just the best b: [1, 3] c: test: 1 ================================================ FILE: examples/empty-no-comment.yaml ================================================ ================================================ FILE: examples/empty.yaml ================================================ # comment ================================================ FILE: examples/environment.yq ================================================ #! yq .[] |( ( select(kind == "scalar") | key + "='" + . + "'"), ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")") ) ================================================ FILE: examples/example.properties ================================================ # comments on values appear person.name = Mike # comments on array values appear person.pets.0 = cat person.food.0 = pizza ================================================ FILE: examples/front-matter.yaml ================================================ --- a: apple b: banana --- hello there apples: great ================================================ FILE: examples/instruction_sample.yaml ================================================ - command: update path: b.c value: #great things: frog # wow! - command: delete path: b.d ================================================ FILE: examples/kyaml.yml ================================================ # leading a: 1 # a line # head b b: 2 c: # head d - d # d line ================================================ FILE: examples/leading-separator.yaml ================================================ --- a: test ================================================ FILE: examples/merge-anchor.yaml ================================================ foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b <<: [*foo,*bar] c: foobarList_c foobar: c: foobar_c <<: *foo thing: foobar_thing ================================================ FILE: examples/mike.xml ================================================ boing ================================================ FILE: examples/mike2.xml ================================================ cool ba2234r bar2234233 ================================================ FILE: examples/multiline-text.yaml ================================================ test: | abcdefg hijklmno ================================================ FILE: examples/multiple_docs.yaml ================================================ commonKey: first document a: Easy! as one two three b: c: 2 d: [3, 4] e: - name: fred value: 3 - name: sam value: 4 --- commonKey: second document another: document: here --- commonKey: third document wow: - here is another ================================================ FILE: examples/multiple_docs_small.yaml ================================================ a: Easy! as one two three --- another: document: here --- - 1 - 2 ================================================ FILE: examples/numbered_keys.yml ================================================ 5: 6: camel! ================================================ FILE: examples/order.yaml ================================================ version: 3 application: MyApp ================================================ FILE: examples/order.yml ================================================ version: '2' services: test: image: ubuntu:14.04 stdin_open: true tty: true ================================================ FILE: examples/sample.hcl ================================================ # Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message) ================================================ FILE: examples/sample.ini ================================================ ; This is a INI document [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 [database] db_host = "localhost" db_port = 5432 db_user = "postgres" db_password = "" db_name = "postgres" ================================================ FILE: examples/sample.json ================================================ {"a":"Easy! as one two three","b":{"c":2,"d":[3,4],"e":[{"name":"fred","value":3},{"name":"sam","value":4}]},"ab":"must appear last"} ================================================ FILE: examples/sample.tf ================================================ # main.tf # Define required providers and minimum Terraform version terraform { required_providers { aws = { source = "hashicorp/aws" version = "~> 5.0" } } required_version = ">= 1.2" } # Configure the AWS provider provider "aws" { region = var.aws_region } # Define an S3 bucket resource resource "aws_s3_bucket" "example_bucket" { bucket = var.bucket_name tags = { Environment = "Development" Project = "TerraformExample" } } ================================================ FILE: examples/sample.toml ================================================ # This is a TOML document title = "TOML Example" [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 [database] enabled = true ports = [ 8000, 8001, 8002 ] data = [ ["delta", "phi"], [3.14] ] temp_targets = { cpu = 79.5, case = 72.0 } [servers] [servers.alpha] ip = "10.0.0.1" role = "frontend" [servers.beta] ip = "10.0.0.2" role = "backend" ================================================ FILE: examples/sample.yaml ================================================ # things a: apple ================================================ FILE: examples/sample2.hcl ================================================ # Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message) ================================================ FILE: examples/sample_array.yaml ================================================ [1,2,3] ================================================ FILE: examples/sample_array_2.yaml ================================================ - 4 - 5 ================================================ FILE: examples/sample_no_sections.ini ================================================ ; This is a INI document name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 enabled = true ip = "10.0.0.1" role = "frontend" treads = 4 ================================================ FILE: examples/sample_objects.csv ================================================ name,numberOfCats,likesApples,height ,1,true,168.8 Samantha's Rabbit,2,false,-188.8 ================================================ FILE: examples/sample_text.yaml ================================================ hi ================================================ FILE: examples/simple-anchor-exploded.yaml ================================================ foo: a: 1 foobar: a: 1 ================================================ FILE: examples/simple-anchor.yaml ================================================ foo: &foo a: 1 foobar: <<: *foo ================================================ FILE: examples/small.properties ================================================ this.is = a properties file ================================================ FILE: examples/small.xml ================================================ is some xml ================================================ FILE: examples/small.yaml ================================================ --- # comment # about things a: cat ================================================ FILE: examples/thing.yml ================================================ a: apple is included b: cool. ================================================ FILE: github-action/Dockerfile ================================================ FROM mikefarah/yq:4 COPY entrypoint.sh /entrypoint.sh # github action recommendation is to run as root. # https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user USER root ENTRYPOINT ["/entrypoint.sh"] ================================================ FILE: github-action/entrypoint.sh ================================================ #!/bin/sh -l set -e echo "::debug::\$cmd: $1" RESULT=$(eval "$1") echo "::debug::\$RESULT: $RESULT" # updating from # https://github.com/orgs/community/discussions/26288#discussioncomment-3876281 # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter delimiter=$(cat /proc/sys/kernel/random/uuid) echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}" echo "${RESULT}" >> "${GITHUB_OUTPUT}" echo "${delimiter}" >> "${GITHUB_OUTPUT}" ================================================ FILE: go.mod ================================================ module github.com/mikefarah/yq/v4 require ( github.com/a8m/envsubst v1.4.3 github.com/alecthomas/participle/v2 v2.1.4 github.com/alecthomas/repr v0.5.2 github.com/dimchansky/utfbom v1.1.1 github.com/elliotchance/orderedmap v1.8.0 github.com/fatih/color v1.18.0 github.com/go-ini/ini v1.67.0 github.com/goccy/go-json v0.10.5 github.com/goccy/go-yaml v1.19.2 github.com/hashicorp/hcl/v2 v2.24.0 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.10 github.com/pelletier/go-toml/v2 v2.2.4 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.1 github.com/zclconf/go-cty v1.17.0 go.yaml.in/yaml/v4 v4.0.0-rc.3 golang.org/x/mod v0.33.0 golang.org/x/net v0.50.0 golang.org/x/text v0.34.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 ) require ( github.com/agext/levenshtein v1.2.1 // indirect github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/tools v0.41.0 // indirect ) go 1.24.0 toolchain go1.24.1 ================================================ FILE: go.sum ================================================ github.com/a8m/envsubst v1.4.3 h1:kDF7paGK8QACWYaQo6KtyYBozY2jhQrTuNNuUxQkhJY= github.com/a8m/envsubst v1.4.3/go.mod h1:4jjHWQlZoaXPoLQUb7H2qT4iLkZDdmEQiOUogdUmqVU= github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8= github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0= github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw= github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE= github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0= github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= ================================================ FILE: go_install_test.go ================================================ package main import ( "io" "testing" "golang.org/x/mod/module" "golang.org/x/mod/zip" ) // TestGoInstallCompatibility ensures the module can be zipped for go install. // This is an integration test that uses the same zip.CreateFromDir function // that go install uses internally. If this test fails, go install will fail. // See: https://github.com/mikefarah/yq/issues/2587 func TestGoInstallCompatibility(t *testing.T) { mod := module.Version{ Path: "github.com/mikefarah/yq/v4", Version: "v4.0.0", // the actual version doesn't matter for validation } if err := zip.CreateFromDir(io.Discard, mod, "."); err != nil { t.Fatalf("Module cannot be zipped for go install: %v", err) } } ================================================ FILE: how-it-works.md ================================================ # Expression Syntax: A Visual Guide In `yq`, expressions are made up of operators and pipes. A context of nodes is passed through the expression, and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. Let's break down the process step by step using a diagram. We'll start with a single YAML document, apply an expression, and observe how the context changes at each step. Given a document like: ```yaml root: items: - name: apple type: fruit - name: carrot type: vegetable - name: banana type: fruit ``` You can use dot notation to access nested structures. For example, to access the `name` of the first item, you would use the expression `.root.items[0].name`, which would return `apple`. But lets see how we could find all the fruit under `items` ## Step 1: Initial Context The context starts at the root of the YAML document. In this case, the entire document is the initial context. ``` root └── items ├── name: apple │ type: fruit ├── name: carrot │ type: vegetable └── name: banana type: fruit ``` ## Step 2: Splatting the Array Using the expression `.root.items[]`, we "splat" the items array. This means each element of the array becomes its own node in the context: ``` Node 1: { name: apple, type: fruit } Node 2: { name: carrot, type: vegetable } Node 3: { name: banana, type: fruit } ``` ## Step 3: Filtering the Nodes Next, we apply a filter to select only the nodes where type is fruit. The expression `.root.items[] | select(.type == "fruit")` filters the nodes: ``` Filtered Node 1: { name: apple, type: fruit } Filtered Node 2: { name: banana, type: fruit } ``` ## Step 4: Extracting a Field Finally, we extract the name field from the filtered nodes using `.root.items[] | select(.type == "fruit") | .name` This results in: ``` apple banana ``` ## Simple assignment example Given a document like: ```yaml a: cat b: dog ``` with an expression: ``` .a = .b ``` Like math expressions - operator precedence is important. The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node ```yaml cat ``` Side note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information. The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node ```yaml dog ``` Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context: ```yaml a: dog b: dog ``` ## Complex assignment, operator precedence rules Just like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. Most of the time, this is intuitively what you'd want, for instance `.a = "cat" | .b = "dog"` is effectively: `(.a = "cat") | (.b = "dog")`. However, this is not always the case, particularly if you have a complex LHS or RHS expression, for instance if you want to select particular nodes to update. Lets say you had: ```yaml - name: bob fruit: apple - name: sally fruit: orange ``` Lets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is: `.[] | select(.name == "sally") | .fruit = "mango"`. Because `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned: ```yaml name: sally fruit: mango ``` **Important**: To properly update this YAML, you must wrap the entire LHS in parentheses. Think of it like using brackets in math to ensure the correct order of operations. `(.[] | select(.name == "sally") | .fruit) = "mango"` Now that entire LHS expression is passed to the 'assign' (`=`) operator, and the yaml is correctly updated and returned: ```yaml - name: bob fruit: apple - name: sally fruit: mango ``` ## Relative update (e.g. `|=`) There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression. In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example. Given a document like: ```yaml a: 1 b: thing ``` with an expression: ``` .a |= . + 1 ``` Similar to the `=` operator, `|=` takes two operands, the LHS and RHS. It pipes the current context (the whole document) through the LHS expression of `.a` to get the node value: ``` 1 ``` Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield: ``` 2 ``` The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context: ```yaml a: 2 b: thing ``` ================================================ FILE: mkdocs.yml ================================================ docs_dir: mkdocs site_dir: docs site_name: Yq theme: 'material' repo_name: 'mikefarah/yq' repo_url: 'https://github.com/mikefarah/yq' ================================================ FILE: pkg/yqlib/all_at_once_evaluator.go ================================================ package yqlib import ( "container/list" ) // A yaml expression evaluator that runs the expression once against all files/nodes in memory. type Evaluator interface { EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error // EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) // EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error) } type allAtOnceEvaluator struct { treeNavigator DataTreeNavigator } func NewAllAtOnceEvaluator() Evaluator { InitExpressionParser() return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()} } func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) { inputCandidates := list.New() for _, node := range nodes { inputCandidates.PushBack(node) } return e.EvaluateCandidateNodes(expression, inputCandidates) } func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) { node, err := ExpressionParser.ParseExpression(expression) if err != nil { return nil, err } context, err := e.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputCandidates}, node) if err != nil { return nil, err } return context.MatchingNodes, nil } func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { fileIndex := 0 var allDocuments = list.New() for _, filename := range filenames { reader, err := readStream(filename) if err != nil { return err } fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) if err != nil { return err } allDocuments.PushBackList(fileDocuments) fileIndex = fileIndex + 1 } if allDocuments.Len() == 0 { candidateNode := createScalarNode(nil, "") allDocuments.PushBack(candidateNode) } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) if err != nil { return err } return printer.PrintResults(matches) } ================================================ FILE: pkg/yqlib/all_at_once_evaluator_test.go ================================================ package yqlib import ( "bufio" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) var evaluateNodesScenario = []expressionScenario{ { document: `a: hello`, expression: `.a`, expected: []string{ "D0, P[a], (!!str)::hello\n", }, }, { document: `a: hello`, expression: `.`, expected: []string{ "D0, P[], (!!map)::a: hello\n", }, }, { document: `- a: "yes"`, expression: `.[] | has("a")`, expected: []string{ "D0, P[0], (!!bool)::true\n", }, }, } func TestAllAtOnceEvaluateNodes(t *testing.T) { var evaluator = NewAllAtOnceEvaluator() // logging.SetLevel(logging.DEBUG, "") for _, tt := range evaluateNodesScenario { decoder := NewYamlDecoder(ConfiguredYamlPreferences) reader := bufio.NewReader(strings.NewReader(tt.document)) err := decoder.Init(reader) if err != nil { t.Error(err) return } candidateNode, errorReading := decoder.Decode() if errorReading != nil { t.Error(errorReading) return } list, _ := evaluator.EvaluateNodes(tt.expression, candidateNode) test.AssertResultComplex(t, tt.expected, resultsToString(t, list)) } } ================================================ FILE: pkg/yqlib/base64_test.go ================================================ //go:build !yq_nobase64 package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const base64EncodedSimple = "YSBzcGVjaWFsIHN0cmluZw==" const base64DecodedSimpleExtraSpaces = "\n " + base64EncodedSimple + " \n" const base64DecodedSimple = "a special string" const base64EncodedUTF8 = "V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==" const base64DecodedUTF8 = "Works with UTF-16 😊" const base64EncodedYaml = "YTogYXBwbGUK" const base64DecodedYaml = "a: apple\n" const base64EncodedEmpty = "" const base64DecodedEmpty = "" const base64MissingPadding = "Y2F0cw" const base64DecodedMissingPadding = "cats" const base64EncodedCats = "Y2F0cw==" const base64DecodedCats = "cats" var base64Scenarios = []formatScenario{ { skipDoc: true, description: "empty decode", input: base64EncodedEmpty, expected: base64DecodedEmpty + "\n", scenarioType: "decode", }, { skipDoc: true, description: "simple decode", input: base64EncodedSimple, expected: base64DecodedSimple + "\n", scenarioType: "decode", }, { description: "Decode base64: simple", subdescription: "Decoded data is assumed to be a string.", input: base64EncodedSimple, expected: base64DecodedSimple + "\n", scenarioType: "decode", }, { description: "Decode base64: UTF-8", subdescription: "Base64 decoding supports UTF-8 encoded strings.", input: base64EncodedUTF8, expected: base64DecodedUTF8 + "\n", scenarioType: "decode", }, { skipDoc: true, description: "decode missing padding", input: base64MissingPadding, expected: base64DecodedMissingPadding + "\n", scenarioType: "decode", }, { description: "Decode with extra spaces", subdescription: "Extra leading/trailing whitespace is stripped", input: base64DecodedSimpleExtraSpaces, expected: base64DecodedSimple + "\n", scenarioType: "decode", }, { skipDoc: true, description: "decode with padding", input: base64EncodedCats, expected: base64DecodedCats + "\n", scenarioType: "decode", }, { skipDoc: true, description: "decode yaml document", input: base64EncodedYaml, expected: base64DecodedYaml + "\n", scenarioType: "decode", }, { description: "Encode base64: string", input: "\"" + base64DecodedSimple + "\"", expected: base64EncodedSimple, scenarioType: "encode", }, { description: "Encode base64: string from document", subdescription: "Extract a string field and encode it to base64.", input: "coolData: \"" + base64DecodedSimple + "\"", expression: ".coolData", expected: base64EncodedSimple, scenarioType: "encode", }, { skipDoc: true, description: "encode empty string", input: "\"\"", expected: "", scenarioType: "encode", }, { skipDoc: true, description: "encode UTF-8 string", input: "\"" + base64DecodedUTF8 + "\"", expected: base64EncodedUTF8, scenarioType: "encode", }, { skipDoc: true, description: "encode cats", input: "\"" + base64DecodedCats + "\"", expected: base64EncodedCats, scenarioType: "encode", }, { description: "Roundtrip: simple", skipDoc: true, input: base64EncodedSimple, expected: base64EncodedSimple, scenarioType: "roundtrip", }, { description: "Roundtrip: UTF-8", skipDoc: true, input: base64EncodedUTF8, expected: base64EncodedUTF8, scenarioType: "roundtrip", }, { description: "Roundtrip: missing padding", skipDoc: true, input: base64MissingPadding, expected: base64EncodedCats, scenarioType: "roundtrip", }, { description: "Roundtrip: empty", skipDoc: true, input: base64EncodedEmpty, expected: base64EncodedEmpty, scenarioType: "roundtrip", }, { description: "Encode error: non-string", skipDoc: true, input: "123", expectedError: "cannot encode !!int as base64, can only operate on strings", scenarioType: "encode-error", }, { description: "Encode error: array", skipDoc: true, input: "[1, 2, 3]", expectedError: "cannot encode !!seq as base64, can only operate on strings", scenarioType: "encode-error", }, { description: "Encode error: map", skipDoc: true, input: "{b: c}", expectedError: "cannot encode !!map as base64, can only operate on strings", scenarioType: "encode-error", }, } func testBase64Scenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": yamlPrefs := ConfiguredYamlPreferences.Copy() yamlPrefs.Indent = 4 test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(yamlPrefs)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder()), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewBase64Encoder()), s.description) case "encode-error": result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder()) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentBase64Scenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "decode": documentBase64DecodeScenario(w, s) case "encode": documentBase64EncodeScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentBase64DecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.txt file of:\n") writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=base64 -oy '%v' sample.txt\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentBase64EncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=base64 '%v' sample.yml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder()))) } func TestBase64Scenarios(t *testing.T) { for _, tt := range base64Scenarios { testBase64Scenario(t, tt) } genericScenarios := make([]interface{}, len(base64Scenarios)) for i, s := range base64Scenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "base64", genericScenarios, documentBase64Scenario) } ================================================ FILE: pkg/yqlib/candidate_node.go ================================================ package yqlib import ( "container/list" "fmt" "strconv" "strings" ) type Kind uint32 const ( SequenceNode Kind = 1 << iota MappingNode ScalarNode AliasNode ) type Style uint32 const ( TaggedStyle Style = 1 << iota DoubleQuotedStyle SingleQuotedStyle LiteralStyle FoldedStyle FlowStyle ) func createStringScalarNode(stringValue string) *CandidateNode { var node = &CandidateNode{Kind: ScalarNode} node.Value = stringValue node.Tag = "!!str" return node } func createScalarNode(value interface{}, stringValue string) *CandidateNode { var node = &CandidateNode{Kind: ScalarNode} node.Value = stringValue switch value.(type) { case float32, float64: node.Tag = "!!float" case int, int64, int32: node.Tag = "!!int" case bool: node.Tag = "!!bool" case string: node.Tag = "!!str" case nil: node.Tag = "!!null" } return node } type NodeInfo struct { Kind string `yaml:"kind"` Style string `yaml:"style,omitempty"` Anchor string `yaml:"anchor,omitempty"` Tag string `yaml:"tag,omitempty"` HeadComment string `yaml:"headComment,omitempty"` LineComment string `yaml:"lineComment,omitempty"` FootComment string `yaml:"footComment,omitempty"` Value string `yaml:"value,omitempty"` Line int `yaml:"line,omitempty"` Column int `yaml:"column,omitempty"` Content []*NodeInfo `yaml:"content,omitempty"` } type CandidateNode struct { Kind Kind Style Style Tag string Value string Anchor string Alias *CandidateNode Content []*CandidateNode HeadComment string LineComment string FootComment string Parent *CandidateNode // parent node Key *CandidateNode // node key, if this is a value from a map (or index in an array) LeadingContent string document uint // the document index of this node filename string Line int Column int fileIndex int // when performing op against all nodes given, this will treat all the nodes as one // (e.g. top level cross document merge). This property does not propagate to child nodes. EvaluateTogether bool IsMapKey bool // For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables // rather than consolidated into nested mappings (default behaviour) EncodeSeparate bool } func (n *CandidateNode) CreateChild() *CandidateNode { return &CandidateNode{ Parent: n, } } func (n *CandidateNode) SetDocument(idx uint) { n.document = idx } func (n *CandidateNode) GetDocument() uint { // defer to parent if n.Parent != nil { return n.Parent.GetDocument() } return n.document } func (n *CandidateNode) SetFilename(name string) { n.filename = name } func (n *CandidateNode) GetFilename() string { if n.Parent != nil { return n.Parent.GetFilename() } return n.filename } func (n *CandidateNode) SetFileIndex(idx int) { n.fileIndex = idx } func (n *CandidateNode) GetFileIndex() int { if n.Parent != nil { return n.Parent.GetFileIndex() } return n.fileIndex } func (n *CandidateNode) GetKey() string { keyPrefix := "" if n.IsMapKey { keyPrefix = fmt.Sprintf("key-%v-", n.Value) } key := "" if n.Key != nil { key = n.Key.Value } return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key) } func (n *CandidateNode) getParsedKey() interface{} { if n.IsMapKey { return n.Value } if n.Key == nil { return nil } if n.Key.Tag == "!!str" { return n.Key.Value } index, err := parseInt(n.Key.Value) if err != nil { return n.Key.Value } return index } func (n *CandidateNode) FilterMapContentByKey(keyPredicate func(*CandidateNode) bool) []*CandidateNode { var result []*CandidateNode for index := 0; index < len(n.Content); index = index + 2 { keyNode := n.Content[index] valueNode := n.Content[index+1] if keyPredicate(keyNode) { result = append(result, keyNode, valueNode) } } return result } func (n *CandidateNode) GetPath() []interface{} { key := n.getParsedKey() if n.Parent != nil && key != nil { return append(n.Parent.GetPath(), key) } if key != nil { return []interface{}{key} } return make([]interface{}, 0) } func (n *CandidateNode) GetNicePath() string { var sb strings.Builder path := n.GetPath() for i, element := range path { elementStr := fmt.Sprintf("%v", element) switch element.(type) { case int: sb.WriteString("[" + elementStr + "]") default: if i == 0 { sb.WriteString(elementStr) } else if strings.ContainsRune(elementStr, '.') { sb.WriteString("[" + elementStr + "]") } else { sb.WriteString("." + elementStr) } } } return sb.String() } func (n *CandidateNode) AsList() *list.List { elMap := list.New() elMap.PushBack(n) return elMap } func (n *CandidateNode) SetParent(parent *CandidateNode) { n.Parent = parent } type ValueVisitor func(*CandidateNode) error func (n *CandidateNode) VisitValues(visitor ValueVisitor) error { switch n.Kind { case MappingNode: for i := 1; i < len(n.Content); i = i + 2 { if err := visitor(n.Content[i]); err != nil { return err } } case SequenceNode: for i := 0; i < len(n.Content); i = i + 1 { if err := visitor(n.Content[i]); err != nil { return err } } } return nil } func (n *CandidateNode) CanVisitValues() bool { return n.Kind == MappingNode || n.Kind == SequenceNode } func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) { key := rawKey.Copy() key.SetParent(n) key.IsMapKey = true value := rawValue.Copy() value.SetParent(n) value.IsMapKey = false // force this, incase we are creating a value from a key value.Key = key n.Content = append(n.Content, key, value) return key, value } func (n *CandidateNode) AddChild(rawChild *CandidateNode) { value := rawChild.Copy() value.SetParent(n) value.IsMapKey = false index := len(n.Content) keyNode := createScalarNode(index, fmt.Sprintf("%v", index)) keyNode.SetParent(n) value.Key = keyNode n.Content = append(n.Content, value) } func (n *CandidateNode) AddChildren(children []*CandidateNode) { if n.Kind == MappingNode { for i := 0; i < len(children); i += 2 { key := children[i] value := children[i+1] n.AddKeyValueChild(key, value) } } else { for _, rawChild := range children { n.AddChild(rawChild) } } } func (n *CandidateNode) GetValueRep() (interface{}, error) { log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value) realTag := n.guessTagFromCustomType() switch realTag { case "!!int": _, val, err := parseInt64(n.Value) return val, err case "!!float": // need to test this return strconv.ParseFloat(n.Value, 64) case "!!bool": return isTruthyNode(n), nil case "!!null": return nil, nil } return n.Value, nil } func (n *CandidateNode) guessTagFromCustomType() string { if strings.HasPrefix(n.Tag, "!!") { return n.Tag } else if n.Value == "" { log.Debug("guessTagFromCustomType: node has no value to guess the type with") return n.Tag } dataBucket, errorReading := parseSnippet(n.Value) if errorReading != nil { log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading) return n.Tag } guessedTag := dataBucket.Tag log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag) return guessedTag } func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode { node := &CandidateNode{ Kind: kind, Tag: tag, Value: value, } return n.CopyAsReplacement(node) } func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode { newCopy := replacement.Copy() newCopy.Parent = n.Parent if n.IsMapKey { newCopy.Key = n } else { newCopy.Key = n.Key } return newCopy } func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode { replacement := n.CreateReplacement(kind, tag, "") replacement.LeadingContent = n.LeadingContent replacement.HeadComment = n.HeadComment replacement.LineComment = n.LineComment replacement.FootComment = n.FootComment replacement.Style = style return replacement } func (n *CandidateNode) Copy() *CandidateNode { return n.doCopy(true) } func (n *CandidateNode) CopyWithoutContent() *CandidateNode { return n.doCopy(false) } func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode { var content []*CandidateNode var copyKey *CandidateNode if n.Key != nil { copyKey = n.Key.Copy() } clone := &CandidateNode{ Kind: n.Kind, Style: n.Style, Tag: n.Tag, Value: n.Value, Anchor: n.Anchor, // ok not to clone this, // as its a reference to somewhere else. Alias: n.Alias, Content: content, HeadComment: n.HeadComment, LineComment: n.LineComment, FootComment: n.FootComment, Parent: n.Parent, Key: copyKey, LeadingContent: n.LeadingContent, document: n.document, filename: n.filename, fileIndex: n.fileIndex, Line: n.Line, Column: n.Column, EvaluateTogether: n.EvaluateTogether, IsMapKey: n.IsMapKey, EncodeSeparate: n.EncodeSeparate, } if cloneContent { clone.AddChildren(n.Content) } return clone } // updates this candidate from the given candidate node func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) { if n == other { log.Debugf("UpdateFrom, no need to update from myself.") return } // if this is an empty map or empty array, use the style of other node. if (n.Kind != ScalarNode && len(n.Content) == 0) || // if the tag has changed (e.g. from str to bool) (n.guessTagFromCustomType() != other.guessTagFromCustomType()) { n.Style = other.Style } n.Content = make([]*CandidateNode, 0) n.Kind = other.Kind n.AddChildren(other.Content) n.Value = other.Value n.UpdateAttributesFrom(other, prefs) } func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) { log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other)) if n.Kind != other.Kind { // clear out the contents when switching to a different type // e.g. map to array n.Content = make([]*CandidateNode, 0) n.Value = "" } n.Kind = other.Kind // don't clobber custom tags... if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" { n.Tag = other.Tag } n.Alias = other.Alias if !prefs.DontOverWriteAnchor { n.Anchor = other.Anchor } // Preserve EncodeSeparate flag for format-specific encoding hints n.EncodeSeparate = other.EncodeSeparate // merge will pickup the style of the new thing // when autocreating nodes if n.Style == 0 { n.Style = other.Style } if other.FootComment != "" { n.FootComment = other.FootComment } if other.HeadComment != "" { n.HeadComment = other.HeadComment } if other.LineComment != "" { n.LineComment = other.LineComment } } func (n *CandidateNode) ConvertToNodeInfo() *NodeInfo { info := &NodeInfo{ Kind: kindToString(n.Kind), Style: styleToString(n.Style), Anchor: n.Anchor, Tag: n.Tag, HeadComment: n.HeadComment, LineComment: n.LineComment, FootComment: n.FootComment, Value: n.Value, Line: n.Line, Column: n.Column, } if len(n.Content) > 0 { info.Content = make([]*NodeInfo, len(n.Content)) for i, child := range n.Content { info.Content[i] = child.ConvertToNodeInfo() } } return info } // Helper functions to convert Kind and Style to string for NodeInfo func kindToString(k Kind) string { switch k { case SequenceNode: return "SequenceNode" case MappingNode: return "MappingNode" case ScalarNode: return "ScalarNode" case AliasNode: return "AliasNode" default: return "Unknown" } } func styleToString(s Style) string { var styles []string if s&TaggedStyle != 0 { styles = append(styles, "TaggedStyle") } if s&DoubleQuotedStyle != 0 { styles = append(styles, "DoubleQuotedStyle") } if s&SingleQuotedStyle != 0 { styles = append(styles, "SingleQuotedStyle") } if s&LiteralStyle != 0 { styles = append(styles, "LiteralStyle") } if s&FoldedStyle != 0 { styles = append(styles, "FoldedStyle") } if s&FlowStyle != 0 { styles = append(styles, "FlowStyle") } return strings.Join(styles, ",") } ================================================ FILE: pkg/yqlib/candidate_node_goccy_yaml.go ================================================ package yqlib import ( "fmt" "strings" yaml "github.com/goccy/go-yaml" "github.com/goccy/go-yaml/ast" goccyToken "github.com/goccy/go-yaml/token" ) func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) (*CandidateNode, error) { newChild := o.CreateChild() err := newChild.UnmarshalGoccyYAML(childNode, cm, anchorMap) return newChild, err } func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error { log.Debugf("UnmarshalYAML %v", node) log.Debugf("UnmarshalYAML %v", node.Type().String()) log.Debugf("UnmarshalYAML Node Value: %v", node.String()) log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment()) if node.GetComment() != nil { commentMapComments := cm[node.GetPath()] for _, comment := range node.GetComment().Comments { // need to use the comment map to find the position :/ log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value) for _, commentMapComment := range commentMapComments { commentMapValue := strings.Join(commentMapComment.Texts, "\n") if commentMapValue == comment.Token.Value { log.Debug("found a matching entry in comment map") // we found the comment in the comment map, // now we can process the position switch commentMapComment.Position { case yaml.CommentHeadPosition: o.HeadComment = comment.String() log.Debug("its a head comment %v", comment.String()) case yaml.CommentLinePosition: o.LineComment = comment.String() log.Debug("its a line comment %v", comment.String()) case yaml.CommentFootPosition: o.FootComment = comment.String() log.Debug("its a foot comment %v", comment.String()) } } } } } o.Value = node.String() o.Line = node.GetToken().Position.Line o.Column = node.GetToken().Position.Column switch node.Type() { case ast.IntegerType: o.Kind = ScalarNode o.Tag = "!!int" case ast.FloatType: o.Kind = ScalarNode o.Tag = "!!float" case ast.BoolType: o.Kind = ScalarNode o.Tag = "!!bool" case ast.NullType: log.Debugf("its a null type with value %v", node.GetToken().Value) o.Kind = ScalarNode o.Tag = "!!null" o.Value = node.GetToken().Value if node.GetToken().Type == goccyToken.ImplicitNullType { o.Value = "" } case ast.StringType: o.Kind = ScalarNode o.Tag = "!!str" switch node.GetToken().Type { case goccyToken.SingleQuoteType: o.Style = SingleQuotedStyle case goccyToken.DoubleQuoteType: o.Style = DoubleQuotedStyle } o.Value = node.(*ast.StringNode).Value log.Debugf("string value %v", node.(*ast.StringNode).Value) case ast.LiteralType: o.Kind = ScalarNode o.Tag = "!!str" o.Style = LiteralStyle astLiteral := node.(*ast.LiteralNode) log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type) if astLiteral.Start.Type == goccyToken.FoldedType { log.Debugf("folded Type %v", astLiteral.Start.Type) o.Style = FoldedStyle } log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value) log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type) // TODO: here I could put the original value with line breaks // to solve the multiline > problem o.Value = astLiteral.Value.Value case ast.TagType: if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm, anchorMap); err != nil { return err } o.Tag = node.(*ast.TagNode).Start.Value case ast.MappingType: log.Debugf("UnmarshalYAML - a mapping node") o.Kind = MappingNode o.Tag = "!!map" mappingNode := node.(*ast.MappingNode) if mappingNode.IsFlowStyle { o.Style = FlowStyle } for _, mappingValueNode := range mappingNode.Values { err := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap) if err != nil { return err } } if mappingNode.FootComment != nil { log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment) o.FootComment = mappingNode.FootComment.String() } case ast.MappingValueType: log.Debugf("UnmarshalYAML - a mapping node") o.Kind = MappingNode o.Tag = "!!map" mappingValueNode := node.(*ast.MappingValueNode) err := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap) if err != nil { return err } case ast.SequenceType: log.Debugf("UnmarshalYAML - a sequence node") o.Kind = SequenceNode o.Tag = "!!seq" sequenceNode := node.(*ast.SequenceNode) if sequenceNode.IsFlowStyle { o.Style = FlowStyle } astSeq := sequenceNode.Values o.Content = make([]*CandidateNode, len(astSeq)) for i := 0; i < len(astSeq); i++ { keyNode := o.CreateChild() keyNode.IsMapKey = true keyNode.Tag = "!!int" keyNode.Kind = ScalarNode keyNode.Value = fmt.Sprintf("%v", i) valueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm, anchorMap) if err != nil { return err } valueNode.Key = keyNode o.Content[i] = valueNode } case ast.AnchorType: log.Debugf("UnmarshalYAML - an anchor node") anchorNode := node.(*ast.AnchorNode) err := o.UnmarshalGoccyYAML(anchorNode.Value, cm, anchorMap) if err != nil { return err } o.Anchor = anchorNode.Name.String() anchorMap[o.Anchor] = o case ast.AliasType: log.Debugf("UnmarshalYAML - an alias node") aliasNode := node.(*ast.AliasNode) o.Kind = AliasNode o.Value = aliasNode.Value.String() o.Alias = anchorMap[o.Value] case ast.MergeKeyType: log.Debugf("UnmarshalYAML - a merge key") o.Kind = ScalarNode o.Tag = "!!merge" // note - I should be able to get rid of this. o.Value = "<<" default: log.Debugf("UnmarshalYAML - no idea of the type!!\n%v: %v", node.Type(), node.String()) } log.Debugf("KIND: %v", o.Kind) return nil } func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error { log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key) // AddKeyValueFirst because it clones the nodes, and we want to have the real refs when Unmarshalling // particularly for the anchorMap keyNode, valueNode := o.AddKeyValueChild(&CandidateNode{}, &CandidateNode{}) if err := keyNode.UnmarshalGoccyYAML(mappingEntry.Key, cm, anchorMap); err != nil { return err } log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value) if err := valueNode.UnmarshalGoccyYAML(mappingEntry.Value, cm, anchorMap); err != nil { return err } if mappingEntry.FootComment != nil { valueNode.FootComment = mappingEntry.FootComment.String() } return nil } ================================================ FILE: pkg/yqlib/candidate_node_test.go ================================================ package yqlib import ( "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) type valueRepScenario struct { input string tag string expected interface{} } var valueRepScenarios = []valueRepScenario{ { input: `"cat"`, expected: `"cat"`, }, { input: `3`, expected: int64(3), }, { input: `3.1`, expected: float64(3.1), }, { input: `true`, expected: true, }, { input: `y`, tag: "!!bool", expected: true, }, { tag: "!!null", expected: nil, }, } func TestCandidateNodeGetValueRepScenarios(t *testing.T) { for _, tt := range valueRepScenarios { node := CandidateNode{Value: tt.input, Tag: tt.tag} actual, err := node.GetValueRep() if err != nil { t.Error(err) return } test.AssertResult(t, tt.expected, actual) } } func TestCandidateNodeChildWhenParentUpdated(t *testing.T) { parent := CandidateNode{} child := parent.CreateChild() parent.SetDocument(1) parent.SetFileIndex(2) parent.SetFilename("meow") test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename") test.AssertResultWithContext(t, 2, child.GetFileIndex(), "file index") test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index") } type createScalarNodeScenario struct { value interface{} stringValue string expectedTag string } var createScalarScenarios = []createScalarNodeScenario{ { value: "mike", stringValue: "mike", expectedTag: "!!str", }, { value: 3, stringValue: "3", expectedTag: "!!int", }, { value: 3.1, stringValue: "3.1", expectedTag: "!!float", }, { value: true, stringValue: "true", expectedTag: "!!bool", }, { value: nil, stringValue: "~", expectedTag: "!!null", }, } func TestCreateScalarNodeScenarios(t *testing.T) { for _, tt := range createScalarScenarios { actual := createScalarNode(tt.value, tt.stringValue) test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue)) test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue)) } } func TestGetKeyForMapValue(t *testing.T) { key := createStringScalarNode("yourKey") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, "3 - yourKey", n.GetKey()) } func TestGetKeyForMapKey(t *testing.T) { key := createStringScalarNode("yourKey") key.IsMapKey = true key.document = 3 test.AssertResult(t, "key-yourKey-3 - ", key.GetKey()) } func TestGetKeyForValue(t *testing.T) { n := CandidateNode{Value: "meow", document: 3} test.AssertResult(t, "3 - ", n.GetKey()) } func TestGetParsedKeyForMapKey(t *testing.T) { key := createStringScalarNode("yourKey") key.IsMapKey = true key.document = 3 test.AssertResult(t, "yourKey", key.getParsedKey()) } func TestGetParsedKeyForLooseValue(t *testing.T) { n := CandidateNode{Value: "meow", document: 3} test.AssertResult(t, nil, n.getParsedKey()) } func TestGetParsedKeyForMapValue(t *testing.T) { key := createStringScalarNode("yourKey") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, "yourKey", n.getParsedKey()) } func TestGetParsedKeyForArrayValue(t *testing.T) { key := createScalarNode(4, "4") n := CandidateNode{Key: key, Value: "meow", document: 3} test.AssertResult(t, 4, n.getParsedKey()) } func TestCandidateNodeAddKeyValueChild(t *testing.T) { key := CandidateNode{Value: "cool", IsMapKey: true} node := CandidateNode{} _, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key) test.AssertResult(t, keyIsValueNow.IsMapKey, false) test.AssertResult(t, key.IsMapKey, true) } func TestConvertToNodeInfo(t *testing.T) { child := &CandidateNode{ Kind: ScalarNode, Style: DoubleQuotedStyle, Tag: "!!str", Value: "childValue", Line: 2, Column: 3, } parent := &CandidateNode{ Kind: MappingNode, Style: TaggedStyle, Tag: "!!map", Value: "", Line: 1, Column: 1, Content: []*CandidateNode{child}, HeadComment: "head", LineComment: "line", FootComment: "foot", Anchor: "anchor", } info := parent.ConvertToNodeInfo() test.AssertResult(t, "MappingNode", info.Kind) test.AssertResult(t, "TaggedStyle", info.Style) test.AssertResult(t, "!!map", info.Tag) test.AssertResult(t, "head", info.HeadComment) test.AssertResult(t, "line", info.LineComment) test.AssertResult(t, "foot", info.FootComment) test.AssertResult(t, "anchor", info.Anchor) test.AssertResult(t, 1, info.Line) test.AssertResult(t, 1, info.Column) if len(info.Content) != 1 { t.Fatalf("Expected 1 child, got %d", len(info.Content)) } childInfo := info.Content[0] test.AssertResult(t, "ScalarNode", childInfo.Kind) test.AssertResult(t, "DoubleQuotedStyle", childInfo.Style) test.AssertResult(t, "!!str", childInfo.Tag) test.AssertResult(t, "childValue", childInfo.Value) test.AssertResult(t, 2, childInfo.Line) test.AssertResult(t, 3, childInfo.Column) } func TestCandidateNodeGetPath(t *testing.T) { // Test root node with no parent root := CandidateNode{Value: "root"} path := root.GetPath() test.AssertResult(t, 0, len(path)) // Test node with key key := createStringScalarNode("myKey") node := CandidateNode{Key: key, Value: "myValue"} path = node.GetPath() test.AssertResult(t, 1, len(path)) test.AssertResult(t, "myKey", path[0]) // Test nested path parent := CandidateNode{} parentKey := createStringScalarNode("parent") parent.Key = parentKey node.Parent = &parent path = node.GetPath() test.AssertResult(t, 2, len(path)) test.AssertResult(t, "parent", path[0]) test.AssertResult(t, "myKey", path[1]) } func TestCandidateNodeGetNicePath(t *testing.T) { // Test simple key key := createStringScalarNode("simple") node := CandidateNode{Key: key} nicePath := node.GetNicePath() test.AssertResult(t, "simple", nicePath) // Test array index arrayKey := createScalarNode(0, "0") arrayNode := CandidateNode{Key: arrayKey} nicePath = arrayNode.GetNicePath() test.AssertResult(t, "[0]", nicePath) dotKey := createStringScalarNode("key.with.dots") dotNode := CandidateNode{Key: dotKey} nicePath = dotNode.GetNicePath() test.AssertResult(t, "key.with.dots", nicePath) // Test nested path parentKey := createStringScalarNode("parent") parent := CandidateNode{Key: parentKey} childKey := createStringScalarNode("child") child := CandidateNode{Key: childKey, Parent: &parent} nicePath = child.GetNicePath() test.AssertResult(t, "parent.child", nicePath) } func TestCandidateNodeFilterMapContentByKey(t *testing.T) { // Create a map with multiple key-value pairs key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") key3 := createStringScalarNode("key3") value3 := createStringScalarNode("value3") mapNode := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key2, value2, key3, value3}, } // Filter by key predicate that matches key1 and key3 filtered := mapNode.FilterMapContentByKey(func(key *CandidateNode) bool { return key.Value == "key1" || key.Value == "key3" }) // Should return key1, value1, key3, value3 test.AssertResult(t, 4, len(filtered)) test.AssertResult(t, "key1", filtered[0].Value) test.AssertResult(t, "value1", filtered[1].Value) test.AssertResult(t, "key3", filtered[2].Value) test.AssertResult(t, "value3", filtered[3].Value) } func TestCandidateNodeVisitValues(t *testing.T) { // Test mapping node key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") mapNode := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key2, value2}, } var visited []string err := mapNode.VisitValues(func(node *CandidateNode) error { visited = append(visited, node.Value) return nil }) test.AssertResult(t, nil, err) test.AssertResult(t, 2, len(visited)) test.AssertResult(t, "value1", visited[0]) test.AssertResult(t, "value2", visited[1]) // Test sequence node item1 := createStringScalarNode("item1") item2 := createStringScalarNode("item2") seqNode := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{item1, item2}, } visited = []string{} err = seqNode.VisitValues(func(node *CandidateNode) error { visited = append(visited, node.Value) return nil }) test.AssertResult(t, nil, err) test.AssertResult(t, 2, len(visited)) test.AssertResult(t, "item1", visited[0]) test.AssertResult(t, "item2", visited[1]) // Test scalar node (should not visit anything) scalarNode := &CandidateNode{ Kind: ScalarNode, Value: "scalar", } visited = []string{} err = scalarNode.VisitValues(func(node *CandidateNode) error { visited = append(visited, node.Value) return nil }) test.AssertResult(t, nil, err) test.AssertResult(t, 0, len(visited)) } func TestCandidateNodeCanVisitValues(t *testing.T) { mapNode := &CandidateNode{Kind: MappingNode} seqNode := &CandidateNode{Kind: SequenceNode} scalarNode := &CandidateNode{Kind: ScalarNode} test.AssertResult(t, true, mapNode.CanVisitValues()) test.AssertResult(t, true, seqNode.CanVisitValues()) test.AssertResult(t, false, scalarNode.CanVisitValues()) } func TestCandidateNodeAddChild(t *testing.T) { parent := &CandidateNode{Kind: SequenceNode} child := createStringScalarNode("child") parent.AddChild(child) test.AssertResult(t, 1, len(parent.Content)) test.AssertResult(t, false, parent.Content[0].IsMapKey) test.AssertResult(t, "0", parent.Content[0].Key.Value) // Check that parent is set correctly if parent.Content[0].Parent != parent { t.Errorf("Expected parent to be set correctly") } } func TestCandidateNodeAddChildren(t *testing.T) { // Test sequence node parent := &CandidateNode{Kind: SequenceNode} child1 := createStringScalarNode("child1") child2 := createStringScalarNode("child2") parent.AddChildren([]*CandidateNode{child1, child2}) test.AssertResult(t, 2, len(parent.Content)) test.AssertResult(t, "child1", parent.Content[0].Value) test.AssertResult(t, "child2", parent.Content[1].Value) // Test mapping node mapParent := &CandidateNode{Kind: MappingNode} key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") mapParent.AddChildren([]*CandidateNode{key1, value1, key2, value2}) test.AssertResult(t, 4, len(mapParent.Content)) test.AssertResult(t, true, mapParent.Content[0].IsMapKey) // key1 test.AssertResult(t, false, mapParent.Content[1].IsMapKey) // value1 test.AssertResult(t, true, mapParent.Content[2].IsMapKey) // key2 test.AssertResult(t, false, mapParent.Content[3].IsMapKey) // value2 } ================================================ FILE: pkg/yqlib/candidate_node_yaml.go ================================================ package yqlib import ( "fmt" yaml "go.yaml.in/yaml/v4" ) func MapYamlStyle(original yaml.Style) Style { switch original { case yaml.TaggedStyle: return TaggedStyle case yaml.DoubleQuotedStyle: return DoubleQuotedStyle case yaml.SingleQuotedStyle: return SingleQuotedStyle case yaml.LiteralStyle: return LiteralStyle case yaml.FoldedStyle: return FoldedStyle case yaml.FlowStyle: return FlowStyle case 0: return 0 } return Style(original) } func MapToYamlStyle(original Style) yaml.Style { switch original { case TaggedStyle: return yaml.TaggedStyle case DoubleQuotedStyle: return yaml.DoubleQuotedStyle case SingleQuotedStyle: return yaml.SingleQuotedStyle case LiteralStyle: return yaml.LiteralStyle case FoldedStyle: return yaml.FoldedStyle case FlowStyle: return yaml.FlowStyle case 0: return 0 } return yaml.Style(original) } func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) { o.Style = MapYamlStyle(node.Style) o.Tag = node.Tag o.Value = node.Value o.Anchor = node.Anchor if o.Anchor != "" { anchorMap[o.Anchor] = o log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o)) } // its a single alias if node.Alias != nil && node.Alias.Anchor != "" { o.Alias = anchorMap[node.Alias.Anchor] log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor])) } o.HeadComment = node.HeadComment o.LineComment = node.LineComment o.FootComment = node.FootComment o.Line = node.Line o.Column = node.Column } func (o *CandidateNode) copyToYamlNode(node *yaml.Node) { node.Style = MapToYamlStyle(o.Style) node.Tag = o.Tag node.Value = o.Value node.Anchor = o.Anchor node.HeadComment = o.HeadComment node.LineComment = o.LineComment node.FootComment = o.FootComment node.Line = o.Line node.Column = o.Column } func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) { newChild := o.CreateChild() // null yaml.Nodes to not end up calling UnmarshalYAML // so we call it explicitly if childNode.Tag == "!!null" { newChild.Kind = ScalarNode newChild.copyFromYamlNode(childNode, anchorMap) return newChild, nil } err := newChild.UnmarshalYAML(childNode, anchorMap) return newChild, err } func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error { log.Debugf("UnmarshalYAML %v", node.Tag) switch node.Kind { case yaml.AliasNode: log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag) o.Kind = AliasNode o.copyFromYamlNode(node, anchorMap) return nil case yaml.ScalarNode: log.Debugf("UnmarshalYAML - a scalar") o.Kind = ScalarNode o.copyFromYamlNode(node, anchorMap) return nil case yaml.MappingNode: log.Debugf("UnmarshalYAML - a mapping node") o.Kind = MappingNode o.copyFromYamlNode(node, anchorMap) o.Content = make([]*CandidateNode, len(node.Content)) for i := 0; i < len(node.Content); i += 2 { keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap) if err != nil { return err } keyNode.IsMapKey = true valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap) if err != nil { return err } valueNode.Key = keyNode o.Content[i] = keyNode o.Content[i+1] = valueNode } log.Debugf("UnmarshalYAML - finished mapping node") return nil case yaml.SequenceNode: log.Debugf("UnmarshalYAML - a sequence: %v", len(node.Content)) o.Kind = SequenceNode o.copyFromYamlNode(node, anchorMap) log.Debugf("node Style: %v", node.Style) log.Debugf("o Style: %v", o.Style) o.Content = make([]*CandidateNode, len(node.Content)) for i := 0; i < len(node.Content); i++ { keyNode := o.CreateChild() keyNode.IsMapKey = true keyNode.Tag = "!!int" keyNode.Kind = ScalarNode keyNode.Value = fmt.Sprintf("%v", i) valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap) if err != nil { return err } valueNode.Key = keyNode o.Content[i] = valueNode } return nil case 0: // not sure when this happens o.copyFromYamlNode(node, anchorMap) log.Debugf("UnmarshalYAML - err.. %v", NodeToString(o)) return nil default: return fmt.Errorf("orderedMap: invalid yaml node") } } func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) { log.Debug("MarshalYAML to yaml: %v", o.Tag) switch o.Kind { case AliasNode: log.Debug("MarshalYAML - alias to yaml: %v", o.Tag) target := &yaml.Node{Kind: yaml.AliasNode} o.copyToYamlNode(target) return target, nil case ScalarNode: log.Debug("MarshalYAML - scalar: %v", o.Value) target := &yaml.Node{Kind: yaml.ScalarNode} o.copyToYamlNode(target) return target, nil case MappingNode, SequenceNode: targetKind := yaml.MappingNode if o.Kind == SequenceNode { targetKind = yaml.SequenceNode } target := &yaml.Node{Kind: targetKind} o.copyToYamlNode(target) log.Debugf("original style: %v", o.Style) log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode) target.Content = make([]*yaml.Node, len(o.Content)) for i := 0; i < len(o.Content); i++ { child, err := o.Content[i].MarshalYAML() if err != nil { return nil, err } target.Content[i] = child } return target, nil } target := &yaml.Node{} o.copyToYamlNode(target) return target, nil } ================================================ FILE: pkg/yqlib/candidiate_node_json.go ================================================ //go:build !yq_nojson package yqlib import ( "bytes" "errors" "fmt" "io" "github.com/goccy/go-json" ) func (o *CandidateNode) setScalarFromJson(value interface{}) error { o.Kind = ScalarNode switch rawData := value.(type) { case nil: o.Tag = "!!null" o.Value = "null" case float64, float32: o.Value = fmt.Sprintf("%v", value) o.Tag = "!!float" // json decoder returns ints as float. if value == float64(int64(rawData.(float64))) { // aha it's an int disguised as a float o.Tag = "!!int" o.Value = fmt.Sprintf("%v", int64(value.(float64))) } case int, int64, int32: o.Value = fmt.Sprintf("%v", value) o.Tag = "!!int" case bool: o.Value = fmt.Sprintf("%v", value) o.Tag = "!!bool" case string: o.Value = rawData o.Tag = "!!str" default: return fmt.Errorf("unrecognised type :( %v", rawData) } return nil } func (o *CandidateNode) UnmarshalJSON(data []byte) error { log.Debug("UnmarshalJSON") switch data[0] { case '{': log.Debug("UnmarshalJSON - its a map!") // its a map o.Kind = MappingNode o.Tag = "!!map" dec := json.NewDecoder(bytes.NewReader(data)) _, err := dec.Token() // open object if err != nil { return err } // cycle through k/v var tok json.Token for tok, err = dec.Token(); err == nil; tok, err = dec.Token() { // we can expect two types: string or Delim. Delim automatically means // that it is the closing bracket of the object, whereas string means // that there is another key. if _, ok := tok.(json.Delim); ok { break } childKey := o.CreateChild() childKey.IsMapKey = true childKey.Value = tok.(string) childKey.Kind = ScalarNode childKey.Tag = "!!str" childValue := o.CreateChild() childValue.Key = childKey if err := dec.Decode(childValue); err != nil { return err } o.Content = append(o.Content, childKey, childValue) } // unexpected error if err != nil && !errors.Is(err, io.EOF) { return err } return nil case '[': o.Kind = SequenceNode o.Tag = "!!seq" log.Debug("UnmarshalJSON - its an array!") var children []*CandidateNode if err := json.Unmarshal(data, &children); err != nil { return err } // now we put the children into the content, and set a key value for them for i, child := range children { if child == nil { // need to represent it as a null scalar child = createScalarNode(nil, "null") } childKey := o.CreateChild() childKey.Kind = ScalarNode childKey.Tag = "!!int" childKey.Value = fmt.Sprintf("%v", i) childKey.IsMapKey = true child.Parent = o child.Key = childKey o.Content = append(o.Content, child) } return nil } log.Debug("UnmarshalJSON - its a scalar!") // otherwise, must be a scalar var scalar interface{} err := json.Unmarshal(data, &scalar) if err != nil { return err } log.Debug("UnmarshalJSON - scalar is %v", scalar) return o.setScalarFromJson(scalar) } func (o *CandidateNode) MarshalJSON() ([]byte, error) { log.Debugf("MarshalJSON %v", NodeToString(o)) buf := new(bytes.Buffer) enc := json.NewEncoder(buf) enc.SetIndent("", " ") enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > switch o.Kind { case AliasNode: log.Debugf("MarshalJSON AliasNode") err := enc.Encode(o.Alias) return buf.Bytes(), err case ScalarNode: log.Debugf("MarshalJSON ScalarNode") value, err := o.GetValueRep() if err != nil { return buf.Bytes(), err } err = enc.Encode(value) return buf.Bytes(), err case MappingNode: log.Debugf("MarshalJSON MappingNode") buf.WriteByte('{') for i := 0; i < len(o.Content); i += 2 { if err := enc.Encode(o.Content[i].Value); err != nil { return nil, err } buf.WriteByte(':') if err := enc.Encode(o.Content[i+1]); err != nil { return nil, err } if i != len(o.Content)-2 { buf.WriteByte(',') } } buf.WriteByte('}') return buf.Bytes(), nil case SequenceNode: log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content)) var err error if len(o.Content) == 0 { buf.WriteString("[]") } else { err = enc.Encode(o.Content) } return buf.Bytes(), err default: err := enc.Encode(nil) return buf.Bytes(), err } } ================================================ FILE: pkg/yqlib/chown_linux.go ================================================ //go:build linux package yqlib import ( "io/fs" "os" "syscall" ) func changeOwner(info fs.FileInfo, file *os.File) error { if stat, ok := info.Sys().(*syscall.Stat_t); ok { uid := int(stat.Uid) gid := int(stat.Gid) err := os.Chown(file.Name(), uid, gid) if err != nil { // this happens with snap confinement // not really a big issue as users can chown // the file themselves if required. log.Info("Skipping chown: %v", err) } } return nil } ================================================ FILE: pkg/yqlib/chown_linux_test.go ================================================ //go:build linux package yqlib import ( "os" "path/filepath" "testing" "time" ) func TestChangeOwner(t *testing.T) { // Create a temporary file for testing tempDir := t.TempDir() testFile := filepath.Join(tempDir, "testfile.txt") // Create a test file err := os.WriteFile(testFile, []byte("test content"), 0600) if err != nil { t.Fatalf("Failed to create test file: %v", err) } // Get file info info, err := os.Stat(testFile) if err != nil { t.Fatalf("Failed to stat test file: %v", err) } // Create another temporary file to change ownership of tempFile, err := os.CreateTemp(tempDir, "chown_test_*.txt") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Test changeOwner function err = changeOwner(info, tempFile) if err != nil { t.Errorf("changeOwner failed: %v", err) } // Verify that the function doesn't panic with valid input tempFile2, err := os.CreateTemp(tempDir, "chown_test2_*.txt") if err != nil { t.Fatalf("Failed to create second temp file: %v", err) } defer os.Remove(tempFile2.Name()) tempFile2.Close() // Test with the second file err = changeOwner(info, tempFile2) if err != nil { t.Errorf("changeOwner failed on second file: %v", err) } } func TestChangeOwnerWithInvalidFileInfo(t *testing.T) { // Create a mock file info that doesn't have syscall.Stat_t mockInfo := &mockFileInfo{ name: "mock", size: 0, mode: 0600, } // Create a temporary file tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Test changeOwner with mock file info (should not panic) err = changeOwner(mockInfo, tempFile) if err != nil { t.Errorf("changeOwner failed with mock file info: %v", err) } } func TestChangeOwnerWithNonExistentFile(t *testing.T) { // Create a temporary file tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Get file info info, err := os.Stat(tempFile.Name()) if err != nil { t.Fatalf("Failed to stat temp file: %v", err) } // Remove the file os.Remove(tempFile.Name()) err = changeOwner(info, tempFile) // The function should not panic even if the file doesn't exist if err != nil { t.Logf("Expected error when changing owner of non-existent file: %v", err) } } // mockFileInfo implements fs.FileInfo but doesn't have syscall.Stat_t type mockFileInfo struct { name string size int64 mode os.FileMode } func (m *mockFileInfo) Name() string { return m.name } func (m *mockFileInfo) Size() int64 { return m.size } func (m *mockFileInfo) Mode() os.FileMode { return m.mode } func (m *mockFileInfo) ModTime() time.Time { return time.Time{} } func (m *mockFileInfo) IsDir() bool { return false } func (m *mockFileInfo) Sys() interface{} { return nil } // This will cause the type assertion to fail func TestChangeOwnerWithSyscallStatT(t *testing.T) { // Create a temporary file tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt") if err != nil { t.Fatalf("Failed to create temp file: %v", err) } defer os.Remove(tempFile.Name()) tempFile.Close() // Get file info info, err := os.Stat(tempFile.Name()) if err != nil { t.Fatalf("Failed to stat temp file: %v", err) } err = changeOwner(info, tempFile) if err != nil { t.Logf("changeOwner returned error (this might be expected in some environments): %v", err) } } ================================================ FILE: pkg/yqlib/chown_not_linux_os.go ================================================ //go:build !linux package yqlib import ( "io/fs" "os" ) func changeOwner(_ fs.FileInfo, _ *os.File) error { return nil } ================================================ FILE: pkg/yqlib/color_print.go ================================================ package yqlib import ( "fmt" "io" "github.com/fatih/color" "github.com/goccy/go-yaml/lexer" "github.com/goccy/go-yaml/printer" ) // Thanks @risentveber! const escape = "\x1b" func format(attr color.Attribute) string { return fmt.Sprintf("%s[%dm", escape, attr) } func colorizeAndPrint(yamlBytes []byte, writer io.Writer) error { tokens := lexer.Tokenize(string(yamlBytes)) var p printer.Printer p.Bool = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgHiMagenta), Suffix: format(color.Reset), } } p.Number = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgHiMagenta), Suffix: format(color.Reset), } } p.MapKey = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgCyan), Suffix: format(color.Reset), } } p.Anchor = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgHiYellow), Suffix: format(color.Reset), } } p.Alias = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgHiYellow), Suffix: format(color.Reset), } } p.String = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgGreen), Suffix: format(color.Reset), } } p.Comment = func() *printer.Property { return &printer.Property{ Prefix: format(color.FgHiBlack), Suffix: format(color.Reset), } } _, err := writer.Write([]byte(p.PrintTokens(tokens) + "\n")) return err } ================================================ FILE: pkg/yqlib/color_print_test.go ================================================ package yqlib import ( "bytes" "strings" "testing" "github.com/fatih/color" ) func TestFormat(t *testing.T) { tests := []struct { name string attr color.Attribute expected string }{ { name: "reset color", attr: color.Reset, expected: "\x1b[0m", }, { name: "red color", attr: color.FgRed, expected: "\x1b[31m", }, { name: "green color", attr: color.FgGreen, expected: "\x1b[32m", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := format(tt.attr) if result != tt.expected { t.Errorf("format(%d) = %q, want %q", tt.attr, result, tt.expected) } }) } } func TestColorizeAndPrint(t *testing.T) { tests := []struct { name string yamlBytes []byte expectErr bool }{ { name: "simple yaml", yamlBytes: []byte("name: test\nage: 25\n"), expectErr: false, }, { name: "yaml with strings", yamlBytes: []byte("name: \"hello world\"\nactive: true\ncount: 42\n"), expectErr: false, }, { name: "yaml with anchors and aliases", yamlBytes: []byte("default: &default\n name: test\nuser: *default\n"), expectErr: false, }, { name: "yaml with comments", yamlBytes: []byte("# This is a comment\nname: test\n"), expectErr: false, }, { name: "empty yaml", yamlBytes: []byte(""), expectErr: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var buf bytes.Buffer err := colorizeAndPrint(tt.yamlBytes, &buf) if tt.expectErr && err == nil { t.Error("Expected error but got none") } if !tt.expectErr && err != nil { t.Errorf("Unexpected error: %v", err) } // Check that output contains escape sequences (color codes) if !tt.expectErr && len(tt.yamlBytes) > 0 { output := buf.String() if !strings.Contains(output, "\x1b[") { t.Error("Expected output to contain color escape sequences") } } }) } } func TestColorizeAndPrintWithDifferentYamlTypes(t *testing.T) { testCases := []struct { name string yaml string expectErr bool }{ { name: "boolean values", yaml: "active: true\ninactive: false\n", }, { name: "numeric values", yaml: "integer: 42\nfloat: 3.14\nnegative: -10\n", }, { name: "map keys", yaml: "user:\n name: john\n age: 30\n", }, { name: "string values", yaml: "message: \"hello world\"\ndescription: 'single quotes'\n", }, { name: "mixed types", yaml: "config:\n debug: true\n port: 8080\n host: \"localhost\"\n", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { var buf bytes.Buffer err := colorizeAndPrint([]byte(tc.yaml), &buf) if tc.expectErr && err == nil { t.Error("Expected error but got none") } if !tc.expectErr && err != nil { t.Errorf("Unexpected error: %v", err) } // Verify output contains color codes if !tc.expectErr { output := buf.String() if !strings.Contains(output, "\x1b[") { t.Error("Expected output to contain color escape sequences") } // Should end with newline if !strings.HasSuffix(output, "\n") { t.Error("Expected output to end with newline") } } }) } } ================================================ FILE: pkg/yqlib/context.go ================================================ package yqlib import ( "container/list" "fmt" "time" logging "gopkg.in/op/go-logging.v1" ) type Context struct { MatchingNodes *list.List Variables map[string]*list.List DontAutoCreate bool datetimeLayout string } func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context { list := list.New() list.PushBack(candidate) newContext := n.ChildContext(list) newContext.DontAutoCreate = true return newContext } func (n *Context) SingleChildContext(candidate *CandidateNode) Context { list := list.New() list.PushBack(candidate) return n.ChildContext(list) } func (n *Context) SetDateTimeLayout(newDateTimeLayout string) { n.datetimeLayout = newDateTimeLayout } func (n *Context) GetDateTimeLayout() string { if n.datetimeLayout != "" { return n.datetimeLayout } return time.RFC3339 } func (n *Context) GetVariable(name string) *list.List { if n.Variables == nil { return nil } return n.Variables[name] } func (n *Context) SetVariable(name string, value *list.List) { if n.Variables == nil { n.Variables = make(map[string]*list.List) } n.Variables[name] = value } func (n *Context) ChildContext(results *list.List) Context { clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout} clone.Variables = make(map[string]*list.List) for variableKey, originalValueList := range n.Variables { variableCopyList := list.New() for el := originalValueList.Front(); el != nil; el = el.Next() { // note that we dont make a copy of the candidate node // this is so the 'ref' operator can work correctly. clonedNode := el.Value.(*CandidateNode) variableCopyList.PushBack(clonedNode) } clone.Variables[variableKey] = variableCopyList } clone.MatchingNodes = results return clone } func (n *Context) ToString() string { if !log.IsEnabledFor(logging.DEBUG) { return "" } result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate) return result + NodesToString(n.MatchingNodes) } func (n *Context) DeepClone() Context { clonedContent := list.New() for el := n.MatchingNodes.Front(); el != nil; el = el.Next() { clonedNode := el.Value.(*CandidateNode).Copy() clonedContent.PushBack(clonedNode) } return n.ChildContext(clonedContent) } func (n *Context) Clone() Context { return n.ChildContext(n.MatchingNodes) } func (n *Context) ReadOnlyClone() Context { clone := n.Clone() clone.DontAutoCreate = true return clone } func (n *Context) WritableClone() Context { clone := n.Clone() clone.DontAutoCreate = false return clone } ================================================ FILE: pkg/yqlib/context_test.go ================================================ package yqlib import ( "container/list" "strings" "testing" "github.com/mikefarah/yq/v4/test" logging "gopkg.in/op/go-logging.v1" ) func TestChildContext(t *testing.T) { expectedOriginal := make(map[string]*list.List) expectedOriginal["dog"] = list.New() expectedOriginal["dog"].PushBack(&CandidateNode{Value: "woof"}) originalVariables := make(map[string]*list.List) originalVariables["dog"] = list.New() originalVariables["dog"].PushBack(&CandidateNode{Value: "woof"}) original := Context{ DontAutoCreate: true, datetimeLayout: "cat", Variables: originalVariables, } newResults := list.New() newResults.PushBack(&CandidateNode{Value: "bar"}) clone := original.ChildContext(newResults) test.AssertResultComplex(t, originalVariables, clone.Variables) clone.Variables["dog"].PushBack("bark") // ensure this is a separate copy test.AssertResultComplex(t, 1, originalVariables["dog"].Len()) } func TestChildContextNoVariables(t *testing.T) { original := Context{ DontAutoCreate: true, datetimeLayout: "cat", } newResults := list.New() newResults.PushBack(&CandidateNode{Value: "bar"}) clone := original.ChildContext(newResults) test.AssertResultComplex(t, make(map[string]*list.List), clone.Variables) } func TestSingleReadonlyChildContext(t *testing.T) { original := Context{ DontAutoCreate: false, datetimeLayout: "2006-01-02", } candidate := &CandidateNode{Value: "test"} clone := original.SingleReadonlyChildContext(candidate) // Should have DontAutoCreate set to true test.AssertResultComplex(t, true, clone.DontAutoCreate) // Should have the candidate node in MatchingNodes test.AssertResultComplex(t, 1, clone.MatchingNodes.Len()) test.AssertResultComplex(t, candidate, clone.MatchingNodes.Front().Value) } func TestSingleChildContext(t *testing.T) { original := Context{ DontAutoCreate: true, datetimeLayout: "2006-01-02", } candidate := &CandidateNode{Value: "test"} clone := original.SingleChildContext(candidate) // Should preserve DontAutoCreate test.AssertResultComplex(t, true, clone.DontAutoCreate) // Should have the candidate node in MatchingNodes test.AssertResultComplex(t, 1, clone.MatchingNodes.Len()) test.AssertResultComplex(t, candidate, clone.MatchingNodes.Front().Value) } func TestSetDateTimeLayout(t *testing.T) { context := Context{} // Test setting datetime layout context.SetDateTimeLayout("2006-01-02T15:04:05Z07:00") test.AssertResultComplex(t, "2006-01-02T15:04:05Z07:00", context.datetimeLayout) } func TestGetDateTimeLayout(t *testing.T) { // Test with custom layout context := Context{datetimeLayout: "2006-01-02"} result := context.GetDateTimeLayout() test.AssertResultComplex(t, "2006-01-02", result) // Test with empty layout (should return default) context = Context{} result = context.GetDateTimeLayout() test.AssertResultComplex(t, "2006-01-02T15:04:05Z07:00", result) } func TestGetVariable(t *testing.T) { // Test with nil Variables context := Context{} result := context.GetVariable("nonexistent") test.AssertResultComplex(t, (*list.List)(nil), result) // Test with existing variable variables := make(map[string]*list.List) variables["test"] = list.New() variables["test"].PushBack(&CandidateNode{Value: "value"}) context = Context{Variables: variables} result = context.GetVariable("test") test.AssertResultComplex(t, variables["test"], result) // Test with non-existent variable result = context.GetVariable("nonexistent") test.AssertResultComplex(t, (*list.List)(nil), result) } func TestSetVariable(t *testing.T) { // Test setting variable when Variables is nil context := Context{} value := list.New() value.PushBack(&CandidateNode{Value: "test"}) context.SetVariable("key", value) test.AssertResultComplex(t, value, context.Variables["key"]) // Test setting variable when Variables already exists context.SetVariable("key2", value) test.AssertResultComplex(t, value, context.Variables["key2"]) } func TestToString(t *testing.T) { context := Context{ DontAutoCreate: true, MatchingNodes: list.New(), } // Add a node to test the full string representation node := &CandidateNode{Value: "test"} context.MatchingNodes.PushBack(node) // Test with debug logging disabled (default) result := context.ToString() test.AssertResultComplex(t, "", result) // Test with debug logging enabled logging.SetLevel(logging.DEBUG, "") defer logging.SetLevel(logging.INFO, "") // Reset to default result2 := context.ToString() test.AssertResultComplex(t, true, len(result2) > 0) test.AssertResultComplex(t, true, strings.Contains(result2, "Context")) test.AssertResultComplex(t, true, strings.Contains(result2, "DontAutoCreate: true")) } func TestDeepClone(t *testing.T) { // Create original context with variables and matching nodes originalVariables := make(map[string]*list.List) originalVariables["test"] = list.New() originalVariables["test"].PushBack(&CandidateNode{Value: "original"}) original := Context{ DontAutoCreate: true, datetimeLayout: "2006-01-02", Variables: originalVariables, MatchingNodes: list.New(), } // Add a node to MatchingNodes node := &CandidateNode{Value: "test"} original.MatchingNodes.PushBack(node) clone := original.DeepClone() // Should preserve DontAutoCreate and datetimeLayout test.AssertResultComplex(t, original.DontAutoCreate, clone.DontAutoCreate) test.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout) // Should have copied variables test.AssertResultComplex(t, 1, len(clone.Variables)) test.AssertResultComplex(t, "original", clone.Variables["test"].Front().Value.(*CandidateNode).Value) // Should have deep copied MatchingNodes test.AssertResultComplex(t, 1, clone.MatchingNodes.Len()) // Verify it's a deep copy by modifying the original original.MatchingNodes.Front().Value.(*CandidateNode).Value = "modified" test.AssertResultComplex(t, "test", clone.MatchingNodes.Front().Value.(*CandidateNode).Value) } func TestClone(t *testing.T) { // Create original context original := Context{ DontAutoCreate: true, datetimeLayout: "2006-01-02", MatchingNodes: list.New(), } node := &CandidateNode{Value: "test"} original.MatchingNodes.PushBack(node) clone := original.Clone() // Should preserve DontAutoCreate and datetimeLayout test.AssertResultComplex(t, original.DontAutoCreate, clone.DontAutoCreate) test.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout) // Should have the same MatchingNodes reference test.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes) } func TestReadOnlyClone(t *testing.T) { original := Context{ DontAutoCreate: false, datetimeLayout: "2006-01-02", MatchingNodes: list.New(), } node := &CandidateNode{Value: "test"} original.MatchingNodes.PushBack(node) clone := original.ReadOnlyClone() // Should set DontAutoCreate to true test.AssertResultComplex(t, true, clone.DontAutoCreate) // Should preserve other fields test.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout) test.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes) } func TestWritableClone(t *testing.T) { original := Context{ DontAutoCreate: true, datetimeLayout: "2006-01-02", MatchingNodes: list.New(), } node := &CandidateNode{Value: "test"} original.MatchingNodes.PushBack(node) clone := original.WritableClone() // Should set DontAutoCreate to false test.AssertResultComplex(t, false, clone.DontAutoCreate) // Should preserve other fields test.AssertResultComplex(t, original.datetimeLayout, clone.datetimeLayout) test.AssertResultComplex(t, original.MatchingNodes, clone.MatchingNodes) } ================================================ FILE: pkg/yqlib/csv.go ================================================ package yqlib type CsvPreferences struct { Separator rune AutoParse bool } func NewDefaultCsvPreferences() CsvPreferences { return CsvPreferences{ Separator: ',', AutoParse: true, } } func NewDefaultTsvPreferences() CsvPreferences { return CsvPreferences{ Separator: '\t', AutoParse: true, } } var ConfiguredCsvPreferences = NewDefaultCsvPreferences() var ConfiguredTsvPreferences = NewDefaultTsvPreferences() ================================================ FILE: pkg/yqlib/csv_test.go ================================================ package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const csvSimple = `name,numberOfCats,likesApples,height Gary,1,true,168.8 Samantha's Rabbit,2,false,-188.8 ` const csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts Gary,1,true,168.8,cool: true Samantha's Rabbit,2,false,-188.8,tall: indeed ` const csvMissing = `name,numberOfCats,likesApples,height ,null,,168.8 ` const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height Gary,3,true,168.8 Samantha's Rabbit,2,false,-188.8 ` const csvSimpleShort = `Name,Number of Cats Gary,1 Samantha's Rabbit,2 ` const tsvSimple = `name numberOfCats likesApples height Gary 1 true 168.8 Samantha's Rabbit 2 false -188.8 ` const expectedYamlFromCSV = `- name: Gary numberOfCats: 1 likesApples: true height: 168.8 - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 ` const expectedYamlFromCSVWithObject = `- name: Gary numberOfCats: 1 likesApples: true height: 168.8 facts: cool: true - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 facts: tall: indeed ` const expectedYamlFromCSVNoParsing = `- name: Gary numberOfCats: 1 likesApples: true height: 168.8 facts: 'cool: true' - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 facts: 'tall: indeed' ` const expectedYamlFromCSVMissingData = `- name: Gary numberOfCats: 1 height: 168.8 - name: Samantha's Rabbit height: -188.8 likesApples: false ` const csvSimpleMissingData = `name,numberOfCats,height Gary,1,168.8 Samantha's Rabbit,,-188.8 ` const csvTestSimpleYaml = `- [i, like, csv] - [because, excel, is, cool]` const expectedSimpleCsv = `i,like,csv because,excel,is,cool ` const tsvTestExpectedSimpleCsv = `i like csv because excel is cool ` var csvScenarios = []formatScenario{ { description: "Encode CSV simple", input: csvTestSimpleYaml, expected: expectedSimpleCsv, scenarioType: "encode-csv", }, { description: "Encode TSV simple", input: csvTestSimpleYaml, expected: tsvTestExpectedSimpleCsv, scenarioType: "encode-tsv", }, { description: "Encode Empty", skipDoc: true, input: `[]`, expected: "", scenarioType: "encode-csv", }, { description: "Comma in value", skipDoc: true, input: `["comma, in, value", things]`, expected: "\"comma, in, value\",things\n", scenarioType: "encode-csv", }, { description: "Encode array of objects to csv", input: expectedYamlFromCSV, expected: csvSimple, scenarioType: "encode-csv", }, { description: "Encode array of objects to custom csv format", subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.", input: expectedYamlFromCSV, expected: csvSimpleShort, expression: `[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]`, scenarioType: "encode-csv", }, { description: "Encode array of objects to csv - missing fields behaviour", subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank", input: expectedYamlFromCSVMissingData, expected: csvSimpleMissingData, scenarioType: "encode-csv", }, { description: "decode csv missing", skipDoc: true, input: csvMissing, expected: csvMissing, scenarioType: "roundtrip-csv", }, { description: "decode csv key", skipDoc: true, input: csvSimple, expression: ".[0].name | key", expected: "name\n", scenarioType: "decode-csv", }, { description: "decode csv parent", skipDoc: true, input: csvSimple, expression: ".[0].name | parent | .height", expected: "168.8\n", scenarioType: "decode-csv", }, { description: "Parse CSV into an array of objects", subdescription: "First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!", input: csvSimpleWithObject, expected: expectedYamlFromCSVWithObject, scenarioType: "decode-csv", }, { description: "Decode CSV line breaks", skipDoc: true, input: "heading1\n\"some data\nwith a line break\"\n", expected: "- heading1: |-\n some data\n with a line break\n", scenarioType: "decode-csv", }, { description: "Parse CSV into an array of objects, no auto-parsing", subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.", input: csvSimpleWithObject, expected: expectedYamlFromCSVNoParsing, scenarioType: "decode-csv-no-auto", }, { description: "values starting with #, no auto parse", skipDoc: true, input: "value\n#ffff", expected: "- value: '#ffff'\n", scenarioType: "decode-csv-no-auto", }, { description: "values starting with #", skipDoc: true, input: "value\n#ffff", expected: "- value: #ffff\n", scenarioType: "decode-csv", }, { description: "Scalar roundtrip", skipDoc: true, input: "mike\ncat", expression: ".[0].mike", expected: "cat\n", scenarioType: "roundtrip-csv", }, { description: "Parse TSV into an array of objects", subdescription: "First row is assumed to be the header row.", input: tsvSimple, expected: expectedYamlFromCSV, scenarioType: "decode-tsv-object", }, { description: "Round trip", input: csvSimple, expected: expectedUpdatedSimpleCsv, expression: `(.[] | select(.name == "Gary") | .numberOfCats) = 3`, scenarioType: "roundtrip-csv", }, } func testCSVScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode-csv": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description) case "encode-tsv": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description) case "decode-csv": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "decode-csv-no-auto": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "decode-tsv-object": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "roundtrip-csv": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType)) writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v sample.%v\n```\n", formatType, formatType)) writeOrPanic(w, "will output\n") separator := ',' if formatType == "tsv" { separator = '\t' } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))), ) } func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s formatScenario, formatType string) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType)) writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v --csv-auto-parse=f sample.%v\n```\n", formatType, formatType)) writeOrPanic(w, "will output\n") separator := ',' if formatType == "tsv" { separator = '\t' } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))), ) } func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v '%v' sample.yml\n```\n", formatType, expression)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v sample.yml\n```\n", formatType)) } writeOrPanic(w, "will output\n") separator := ',' if formatType == "tsv" { separator = '\t' } csvPrefs := NewDefaultCsvPreferences() csvPrefs.Separator = separator writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))), ) } func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType)) writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v '%v' sample.%v\n```\n", formatType, formatType, expression, formatType)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v sample.%v\n```\n", formatType, formatType, formatType)) } writeOrPanic(w, "will output\n") separator := ',' if formatType == "tsv" { separator = '\t' } csvPrefs := NewDefaultCsvPreferences() csvPrefs.Separator = separator writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))), ) } func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "encode-csv": documentCSVEncodeScenario(w, s, "csv") case "encode-tsv": documentCSVEncodeScenario(w, s, "tsv") case "decode-csv": documentCSVDecodeObjectScenario(w, s, "csv") case "decode-csv-no-auto": documentCSVDecodeObjectNoAutoScenario(w, s, "csv") case "decode-tsv-object": documentCSVDecodeObjectScenario(w, s, "tsv") case "roundtrip-csv": documentCSVRoundTripScenario(w, s, "csv") default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func TestCSVScenarios(t *testing.T) { for _, tt := range csvScenarios { testCSVScenario(t, tt) } genericScenarios := make([]interface{}, len(csvScenarios)) for i, s := range csvScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario) } ================================================ FILE: pkg/yqlib/data_tree_navigator.go ================================================ package yqlib import ( "fmt" logging "gopkg.in/op/go-logging.v1" ) type DataTreeNavigator interface { // given the context and an expressionNode, // this will process the against the given expressionNode and return // a new context of matching candidates GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error } type dataTreeNavigator struct { } func NewDataTreeNavigator() DataTreeNavigator { return &dataTreeNavigator{} } func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error { assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} if rhsCandidateNode.Kind == MappingNode { log.Debug("DeeplyAssign: deeply merging object") // if the rhs is a map, we need to deeply merge it in. // otherwise we'll clobber any existing fields assignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{ AppendArrays: true, TraversePrefs: traversePreferences{DontFollowAlias: true}, AssignPrefs: assignPreferences{}, }} } rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode} assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, LHS: createTraversalTree(path, traversePreferences{}, false), RHS: &ExpressionNode{Operation: rhsOp}, } _, err := d.GetMatchingNodes(context, assignmentOpNode) return err } func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) { if expressionNode == nil { log.Debugf("getMatchingNodes - nothing to do") return context, nil } log.Debugf("Processing Op: %v", expressionNode.Operation.toString()) if log.IsEnabledFor(logging.DEBUG) { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { log.Debug(NodeToString(el.Value.(*CandidateNode))) } } handler := expressionNode.Operation.OperationType.Handler if handler != nil { return handler(d, context, expressionNode) } return Context{}, fmt.Errorf("unknown operator %v", expressionNode.Operation.OperationType.Type) } ================================================ FILE: pkg/yqlib/data_tree_navigator_test.go ================================================ package yqlib import ( "container/list" "testing" "github.com/mikefarah/yq/v4/test" ) func TestGetMatchingNodes_NilExpressionNode(t *testing.T) { navigator := NewDataTreeNavigator() context := Context{ MatchingNodes: list.New(), } result, err := navigator.GetMatchingNodes(context, nil) test.AssertResult(t, nil, err) test.AssertResultComplex(t, context, result) } func TestGetMatchingNodes_UnknownOperator(t *testing.T) { navigator := NewDataTreeNavigator() context := Context{ MatchingNodes: list.New(), } // Create an expression node with an unknown operation type unknownOpType := &operationType{Type: "UNKNOWN", Handler: nil} expressionNode := &ExpressionNode{ Operation: &Operation{OperationType: unknownOpType}, } result, err := navigator.GetMatchingNodes(context, expressionNode) test.AssertResult(t, "unknown operator UNKNOWN", err.Error()) test.AssertResultComplex(t, Context{}, result) } func TestGetMatchingNodes_ValidOperator(t *testing.T) { navigator := NewDataTreeNavigator() // Create a simple context with a scalar node scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "test", } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(scalarNode) // Create an expression node with a valid operation (self reference) expressionNode := &ExpressionNode{ Operation: &Operation{OperationType: selfReferenceOpType}, } result, err := navigator.GetMatchingNodes(context, expressionNode) test.AssertResult(t, nil, err) test.AssertResult(t, 1, result.MatchingNodes.Len()) // Verify the result contains the same node resultNode := result.MatchingNodes.Front().Value.(*CandidateNode) test.AssertResult(t, scalarNode, resultNode) } func TestDeeplyAssign_ScalarNode(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "existing", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "old_value"}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a scalar node to assign scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "new_value", } // Assign to path ["new_key"] path := []interface{}{"new_key"} err := navigator.DeeplyAssign(context, path, scalarNode) test.AssertResult(t, nil, err) // Verify the assignment was made // The root node should now have the new key-value pair test.AssertResult(t, 4, len(rootNode.Content)) // 2 original + 2 new // Find the new key-value pair found := false for i := 0; i < len(rootNode.Content)-1; i += 2 { key := rootNode.Content[i] value := rootNode.Content[i+1] if key.Value == "new_key" && value.Value == "new_value" { found = true break } } test.AssertResult(t, true, found) } func TestDeeplyAssign_MappingNode(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "existing", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "old_value"}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a mapping node to assign (this should trigger deep merge) mappingNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "nested_key", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "nested_value"}, }, } // Assign to path ["new_map"] path := []interface{}{"new_map"} err := navigator.DeeplyAssign(context, path, mappingNode) test.AssertResult(t, nil, err) // Verify the assignment was made // The root node should now have the new mapping test.AssertResult(t, 4, len(rootNode.Content)) // 2 original + 2 new // Find the new mapping found := false for i := 0; i < len(rootNode.Content); i += 2 { if i+1 < len(rootNode.Content) { key := rootNode.Content[i] value := rootNode.Content[i+1] if key.Value == "new_map" && value.Kind == MappingNode { found = true // Verify the nested content test.AssertResult(t, 2, len(value.Content)) test.AssertResult(t, "nested_key", value.Content[0].Value) test.AssertResult(t, "nested_value", value.Content[1].Value) break } } } test.AssertResult(t, true, found) } func TestDeeplyAssign_DeepPath(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "level1", IsMapKey: true}, {Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{}}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a scalar node to assign scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "deep_value", } // Assign to deep path ["level1", "level2", "level3"] path := []interface{}{"level1", "level2", "level3"} err := navigator.DeeplyAssign(context, path, scalarNode) test.AssertResult(t, nil, err) // Verify the deep assignment was made level1Node := rootNode.Content[1] // The mapping node test.AssertResult(t, 2, len(level1Node.Content)) // Should have level2 key-value level2Key := level1Node.Content[0] level2Value := level1Node.Content[1] test.AssertResult(t, "level2", level2Key.Value) test.AssertResult(t, MappingNode, level2Value.Kind) level3Key := level2Value.Content[0] level3Value := level2Value.Content[1] test.AssertResult(t, "level3", level3Key.Value) test.AssertResult(t, "deep_value", level3Value.Value) } func TestDeeplyAssign_ArrayPath(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node containing an array rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "array", IsMapKey: true}, {Kind: SequenceNode, Tag: "!!seq", Content: []*CandidateNode{}}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a scalar node to assign scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "array_value", } // Assign to array path ["array", 0] path := []interface{}{"array", 0} err := navigator.DeeplyAssign(context, path, scalarNode) test.AssertResult(t, nil, err) // Verify the array assignment was made arrayNode := rootNode.Content[1] // The sequence node test.AssertResult(t, 1, len(arrayNode.Content)) // Should have one element arrayElement := arrayNode.Content[0] test.AssertResult(t, "array_value", arrayElement.Value) } func TestDeeplyAssign_OverwriteExisting(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "key", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "old_value"}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a scalar node to assign scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "new_value", } // Assign to existing path ["key"] path := []interface{}{"key"} err := navigator.DeeplyAssign(context, path, scalarNode) test.AssertResult(t, nil, err) // Verify the value was overwritten test.AssertResult(t, 2, len(rootNode.Content)) // Should still have 2 elements key := rootNode.Content[0] value := rootNode.Content[1] test.AssertResult(t, "key", key.Value) test.AssertResult(t, "new_value", value.Value) // Should be overwritten } func TestDeeplyAssign_ErrorHandling(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a scalar node (not a mapping) scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "not_a_map", } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(scalarNode) // Create a scalar node to assign assignNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "value", } path := []interface{}{"key"} err := navigator.DeeplyAssign(context, path, assignNode) // Print the actual error for debugging if err != nil { t.Logf("Actual error: %v", err) } test.AssertResult(t, nil, err) } func TestGetMatchingNodes_WithVariables(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with variables variables := make(map[string]*list.List) varList := list.New() varList.PushBack(&CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "var_value"}) variables["test_var"] = varList context := Context{ MatchingNodes: list.New(), Variables: variables, } // Create an expression node that gets a variable expressionNode := &ExpressionNode{ Operation: &Operation{OperationType: getVariableOpType, StringValue: "test_var"}, } result, err := navigator.GetMatchingNodes(context, expressionNode) test.AssertResult(t, nil, err) test.AssertResult(t, 1, result.MatchingNodes.Len()) // Verify the variable was retrieved resultNode := result.MatchingNodes.Front().Value.(*CandidateNode) test.AssertResult(t, "var_value", resultNode.Value) } func TestGetMatchingNodes_EmptyContext(t *testing.T) { navigator := NewDataTreeNavigator() // Create an empty context context := Context{ MatchingNodes: list.New(), } // Create an expression node with self reference expressionNode := &ExpressionNode{ Operation: &Operation{OperationType: selfReferenceOpType}, } result, err := navigator.GetMatchingNodes(context, expressionNode) test.AssertResult(t, nil, err) test.AssertResult(t, 0, result.MatchingNodes.Len()) } func TestDeeplyAssign_ComplexMappingMerge(t *testing.T) { navigator := NewDataTreeNavigator() // Create a context with a root mapping node containing nested data rootNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "config", IsMapKey: true}, {Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "existing_key", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "existing_value"}, }}, }, } context := Context{ MatchingNodes: list.New(), } context.MatchingNodes.PushBack(rootNode) // Create a mapping node to merge mappingNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "new_key", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "new_value"}, {Kind: ScalarNode, Tag: "!!str", Value: "existing_key", IsMapKey: true}, {Kind: ScalarNode, Tag: "!!str", Value: "updated_value"}, }, } // Assign to path ["config"] (should merge with existing mapping) path := []interface{}{"config"} err := navigator.DeeplyAssign(context, path, mappingNode) test.AssertResult(t, nil, err) // Verify the merge was successful configNode := rootNode.Content[1] // The config mapping node test.AssertResult(t, 4, len(configNode.Content)) // Should have 2 key-value pairs // Check that both existing and new keys are present foundExisting := false foundNew := false for i := 0; i < len(configNode.Content); i += 2 { if i+1 < len(configNode.Content) { key := configNode.Content[i] value := configNode.Content[i+1] switch key.Value { case "existing_key": foundExisting = true test.AssertResult(t, "updated_value", value.Value) // Should be updated case "new_key": foundNew = true test.AssertResult(t, "new_value", value.Value) } } } test.AssertResult(t, true, foundExisting) test.AssertResult(t, true, foundNew) } ================================================ FILE: pkg/yqlib/decoder.go ================================================ package yqlib import ( "io" ) type Decoder interface { Init(reader io.Reader) error Decode() (*CandidateNode, error) } ================================================ FILE: pkg/yqlib/decoder_base64.go ================================================ //go:build !yq_nobase64 package yqlib import ( "bytes" "encoding/base64" "io" "strings" ) type base64Decoder struct { reader io.Reader finished bool readAnything bool encoding base64.Encoding } func NewBase64Decoder() Decoder { return &base64Decoder{finished: false, encoding: *base64.StdEncoding} } func (dec *base64Decoder) Init(reader io.Reader) error { // Read all data from the reader and strip leading/trailing whitespace // This is necessary because base64 decoding needs to see the complete input // to handle padding correctly, and we need to strip whitespace before decoding. buf := new(bytes.Buffer) if _, err := buf.ReadFrom(reader); err != nil { return err } // Strip leading and trailing whitespace stripped := strings.TrimSpace(buf.String()) // Add padding if needed (base64 strings should be a multiple of 4 characters) padLen := len(stripped) % 4 if padLen > 0 { stripped += strings.Repeat("=", 4-padLen) } // Create a new reader from the stripped and padded data dec.reader = strings.NewReader(stripped) dec.readAnything = false dec.finished = false return nil } func (dec *base64Decoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } base64Reader := base64.NewDecoder(&dec.encoding, dec.reader) buf := new(bytes.Buffer) if _, err := buf.ReadFrom(base64Reader); err != nil { return nil, err } if buf.Len() == 0 { dec.finished = true // if we've read _only_ an empty string, lets return that // otherwise if we've already read some bytes, and now we get // an empty string, then we are done. if dec.readAnything { return nil, io.EOF } } dec.readAnything = true return createStringScalarNode(buf.String()), nil } ================================================ FILE: pkg/yqlib/decoder_csv_object.go ================================================ //go:build !yq_nocsv package yqlib import ( "encoding/csv" "errors" "io" "github.com/dimchansky/utfbom" ) type csvObjectDecoder struct { prefs CsvPreferences reader csv.Reader finished bool } func NewCSVObjectDecoder(prefs CsvPreferences) Decoder { return &csvObjectDecoder{prefs: prefs} } func (dec *csvObjectDecoder) Init(reader io.Reader) error { cleanReader, enc := utfbom.Skip(reader) log.Debugf("Detected encoding: %s\n", enc) dec.reader = *csv.NewReader(cleanReader) dec.reader.Comma = dec.prefs.Separator dec.finished = false return nil } func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode { node, err := parseSnippet(content) // if we're not auto-parsing, then we wont put in parsed objects or arrays // but we still parse scalars if err != nil || (!dec.prefs.AutoParse && (node.Kind != ScalarNode || node.Value != content)) { return createScalarNode(content, content) } return node } func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *CandidateNode { objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"} for i, header := range headerRow { objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i])) } return objectNode } func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } headerRow, err := dec.reader.Read() log.Debugf(": headerRow%v", headerRow) if err != nil { return nil, err } rootArray := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} contentRow, err := dec.reader.Read() for err == nil && len(contentRow) > 0 { log.Debugf("Adding contentRow: %v", contentRow) rootArray.AddChild(dec.createObject(headerRow, contentRow)) contentRow, err = dec.reader.Read() log.Debugf("Read next contentRow: %v, %v", contentRow, err) } if !errors.Is(err, io.EOF) { return nil, err } return rootArray, nil } ================================================ FILE: pkg/yqlib/decoder_goccy_yaml.go ================================================ //go:build !yq_noyaml // // NOTE this is still a WIP - not yet ready. // package yqlib import ( "io" yaml "github.com/goccy/go-yaml" "github.com/goccy/go-yaml/ast" ) type goccyYamlDecoder struct { decoder yaml.Decoder cm yaml.CommentMap // anchor map persists over multiple documents for convenience. anchorMap map[string]*CandidateNode } func NewGoccyYAMLDecoder() Decoder { return &goccyYamlDecoder{} } func (dec *goccyYamlDecoder) Init(reader io.Reader) error { dec.cm = yaml.CommentMap{} dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap()) dec.anchorMap = make(map[string]*CandidateNode) return nil } func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) { var ast ast.Node err := dec.decoder.Decode(&ast) if err != nil { return nil, err } candidateNode := &CandidateNode{} if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm, dec.anchorMap); err != nil { return nil, err } return candidateNode, nil } ================================================ FILE: pkg/yqlib/decoder_hcl.go ================================================ //go:build !yq_nohcl package yqlib import ( "fmt" "io" "math/big" "sort" "strconv" "strings" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" ) type hclDecoder struct { file *hcl.File fileBytes []byte readAnything bool documentIndex uint } func NewHclDecoder() Decoder { return &hclDecoder{} } // sortedAttributes returns attributes in declaration order by source position func sortedAttributes(attrs hclsyntax.Attributes) []*attributeWithName { var sorted []*attributeWithName for name, attr := range attrs { sorted = append(sorted, &attributeWithName{Name: name, Attr: attr}) } sort.Slice(sorted, func(i, j int) bool { return sorted[i].Attr.Range().Start.Byte < sorted[j].Attr.Range().Start.Byte }) return sorted } type attributeWithName struct { Name string Attr *hclsyntax.Attribute } // extractLineComment extracts any inline comment after the given position func extractLineComment(src []byte, endPos int) string { // Look for # comment after the token for i := endPos; i < len(src); i++ { if src[i] == '#' { // Found comment, extract until end of line start := i for i < len(src) && src[i] != '\n' { i++ } return strings.TrimSpace(string(src[start:i])) } if src[i] == '\n' { // Hit newline before comment break } // Skip whitespace and other characters } return "" } // extractHeadComment extracts comments before a given start position func extractHeadComment(src []byte, startPos int) string { var comments []string // Start just before the token and skip trailing whitespace i := startPos - 1 for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') { i-- } for i >= 0 { // Find line boundaries lineEnd := i for i >= 0 && src[i] != '\n' { i-- } lineStart := i + 1 line := strings.TrimRight(string(src[lineStart:lineEnd+1]), " \t\r") trimmed := strings.TrimSpace(line) if trimmed == "" { break } if !strings.HasPrefix(trimmed, "#") { break } comments = append([]string{trimmed}, comments...) // Move to previous line (skip any whitespace/newlines) i = lineStart - 1 for i >= 0 && (src[i] == ' ' || src[i] == '\t' || src[i] == '\n' || src[i] == '\r') { i-- } } if len(comments) > 0 { return strings.Join(comments, "\n") } return "" } func (dec *hclDecoder) Init(reader io.Reader) error { data, err := io.ReadAll(reader) if err != nil { return err } file, diags := hclsyntax.ParseConfig(data, "input.hcl", hcl.Pos{Line: 1, Column: 1}) if diags != nil && diags.HasErrors() { return fmt.Errorf("hcl parse error: %w", diags) } dec.file = file dec.fileBytes = data dec.readAnything = false dec.documentIndex = 0 return nil } func (dec *hclDecoder) Decode() (*CandidateNode, error) { if dec.readAnything { return nil, io.EOF } dec.readAnything = true if dec.file == nil { return nil, fmt.Errorf("no hcl file parsed") } root := &CandidateNode{Kind: MappingNode} // process attributes in declaration order body := dec.file.Body.(*hclsyntax.Body) firstAttr := true for _, attrWithName := range sortedAttributes(body.Attributes) { keyNode := createStringScalarNode(attrWithName.Name) valNode := convertHclExprToNode(attrWithName.Attr.Expr, dec.fileBytes) // Attach comments if any attrRange := attrWithName.Attr.Range() headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte) if firstAttr && headComment != "" { // For the first attribute, apply its head comment to the root root.HeadComment = headComment firstAttr = false } else if headComment != "" { keyNode.HeadComment = headComment } if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" { valNode.LineComment = lineComment } root.AddKeyValueChild(keyNode, valNode) } // process blocks // Count blocks by type at THIS level to detect multiple separate blocks blocksByType := make(map[string]int) for _, block := range body.Blocks { blocksByType[block.Type]++ } for _, block := range body.Blocks { addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1) } dec.documentIndex++ root.document = dec.documentIndex - 1 return root, nil } func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode { node := &CandidateNode{Kind: MappingNode} for _, attrWithName := range sortedAttributes(body.Attributes) { key := createStringScalarNode(attrWithName.Name) val := convertHclExprToNode(attrWithName.Attr.Expr, src) // Attach comments if any attrRange := attrWithName.Attr.Range() if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" { key.HeadComment = headComment } if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" { val.LineComment = lineComment } node.AddKeyValueChild(key, val) } // Process nested blocks, counting blocks by type at THIS level // to detect which block types appear multiple times blocksByType := make(map[string]int) for _, block := range body.Blocks { blocksByType[block.Type]++ } for _, block := range body.Blocks { addBlockToMapping(node, block, src, blocksByType[block.Type] > 1) } return node } // addBlockToMapping nests block type and labels into the parent mapping, merging children. // isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) { bodyNode := hclBodyToNode(block.Body, src) current := parent // ensure block type mapping exists var typeNode *CandidateNode for i := 0; i < len(current.Content); i += 2 { if current.Content[i].Value == block.Type { typeNode = current.Content[i+1] break } } if typeNode == nil { _, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode}) // Mark the type node if there are multiple blocks of this type at this level // This tells the encoder to emit them as separate blocks rather than consolidating them if isMultipleBlocksOfType { typeNode.EncodeSeparate = true } } current = typeNode // walk labels, creating/merging mappings for _, label := range block.Labels { var next *CandidateNode for i := 0; i < len(current.Content); i += 2 { if current.Content[i].Value == label { next = current.Content[i+1] break } } if next == nil { _, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode}) } current = next } // merge body attributes/blocks into the final mapping for i := 0; i < len(bodyNode.Content); i += 2 { current.AddKeyValueChild(bodyNode.Content[i], bodyNode.Content[i+1]) } } func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode { // handle literal values directly switch e := expr.(type) { case *hclsyntax.LiteralValueExpr: v := e.Val if v.IsNull() { return createScalarNode(nil, "") } switch { case v.Type().Equals(cty.String): // prefer to extract exact source (to avoid extra quoting) when available // Prefer the actual cty string value s := v.AsString() node := createScalarNode(s, s) // Don't set style for regular quoted strings - let YAML handle naturally return node case v.Type().Equals(cty.Bool): b := v.True() return createScalarNode(b, strconv.FormatBool(b)) case v.Type() == cty.Number: // prefer integers when the numeric value is integral bf := v.AsBigFloat() if bf == nil { // fallback to string return createStringScalarNode(v.GoString()) } // check if bf represents an exact integer if intVal, acc := bf.Int(nil); acc == big.Exact { s := intVal.String() return createScalarNode(intVal.Int64(), s) } s := bf.Text('g', -1) return createScalarNode(0.0, s) case v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType(): seq := &CandidateNode{Kind: SequenceNode} it := v.ElementIterator() for it.Next() { _, val := it.Element() // convert cty.Value to a node by wrapping in literal expr via string representation child := convertCtyValueToNode(val) seq.AddChild(child) } return seq case v.Type().IsMapType() || v.Type().IsObjectType(): m := &CandidateNode{Kind: MappingNode} it := v.ElementIterator() for it.Next() { key, val := it.Element() keyStr := key.AsString() keyNode := createStringScalarNode(keyStr) valNode := convertCtyValueToNode(val) m.AddKeyValueChild(keyNode, valNode) } return m default: // fallback to string s := v.GoString() return createStringScalarNode(s) } case *hclsyntax.TupleConsExpr: // parse tuple/list into YAML sequence seq := &CandidateNode{Kind: SequenceNode} for _, exprVal := range e.Exprs { child := convertHclExprToNode(exprVal, src) seq.AddChild(child) } return seq case *hclsyntax.ObjectConsExpr: // parse object into YAML mapping m := &CandidateNode{Kind: MappingNode} m.Style = FlowStyle // Mark as inline object (flow style) for encoder for _, item := range e.Items { // evaluate key expression to get the key string keyVal, keyDiags := item.KeyExpr.Value(nil) if keyDiags != nil && keyDiags.HasErrors() { // fallback: try to extract key from source r := item.KeyExpr.Range() start := r.Start.Byte end := r.End.Byte if start >= 0 && end >= start && end <= len(src) { keyNode := createStringScalarNode(strings.TrimSpace(string(src[start:end]))) valNode := convertHclExprToNode(item.ValueExpr, src) m.AddKeyValueChild(keyNode, valNode) } continue } keyStr := keyVal.AsString() keyNode := createStringScalarNode(keyStr) valNode := convertHclExprToNode(item.ValueExpr, src) m.AddKeyValueChild(keyNode, valNode) } return m case *hclsyntax.TemplateExpr: // Reconstruct template string, preserving ${} syntax for interpolations var parts []string for _, p := range e.Parts { switch lp := p.(type) { case *hclsyntax.LiteralValueExpr: if lp.Val.Type().Equals(cty.String) { parts = append(parts, lp.Val.AsString()) } else { parts = append(parts, lp.Val.GoString()) } default: // Non-literal expression - reconstruct with ${} wrapper r := p.Range() start := r.Start.Byte end := r.End.Byte if start >= 0 && end >= start && end <= len(src) { exprText := string(src[start:end]) parts = append(parts, "${"+exprText+"}") } else { parts = append(parts, fmt.Sprintf("${%v}", p)) } } } combined := strings.Join(parts, "") node := createScalarNode(combined, combined) // Set DoubleQuotedStyle for all templates (which includes all quoted strings in HCL) // This ensures HCL roundtrips preserve quotes, and YAML properly quotes strings with ${} node.Style = DoubleQuotedStyle return node case *hclsyntax.ScopeTraversalExpr: // Simple identifier/traversal (e.g. unquoted string literal in HCL) r := e.Range() start := r.Start.Byte end := r.End.Byte if start >= 0 && end >= start && end <= len(src) { text := strings.TrimSpace(string(src[start:end])) return createStringScalarNode(text) } // Fallback to root name if source unavailable if len(e.Traversal) > 0 { if root, ok := e.Traversal[0].(hcl.TraverseRoot); ok { return createStringScalarNode(root.Name) } } return createStringScalarNode("") case *hclsyntax.FunctionCallExpr: // Preserve function calls as raw expressions for roundtrip r := e.Range() start := r.Start.Byte end := r.End.Byte if start >= 0 && end >= start && end <= len(src) { text := strings.TrimSpace(string(src[start:end])) node := createStringScalarNode(text) node.Style = 0 return node } node := createStringScalarNode(e.Name) node.Style = 0 return node default: // try to evaluate the expression (handles unary, binary ops, etc.) val, diags := expr.Value(nil) if diags == nil || !diags.HasErrors() { // successfully evaluated, convert cty.Value to node return convertCtyValueToNode(val) } // fallback: extract source text for the expression r := expr.Range() start := r.Start.Byte end := r.End.Byte if start >= 0 && end >= start && end <= len(src) { text := string(src[start:end]) // Mark as unquoted expression so encoder emits without quoting node := createStringScalarNode(text) node.Style = 0 return node } return createStringScalarNode(fmt.Sprintf("%v", expr)) } } func convertCtyValueToNode(v cty.Value) *CandidateNode { if v.IsNull() { return createScalarNode(nil, "") } switch { case v.Type().Equals(cty.String): return createScalarNode("", v.AsString()) case v.Type().Equals(cty.Bool): b := v.True() return createScalarNode(b, strconv.FormatBool(b)) case v.Type() == cty.Number: bf := v.AsBigFloat() if bf == nil { return createStringScalarNode(v.GoString()) } if intVal, acc := bf.Int(nil); acc == big.Exact { s := intVal.String() return createScalarNode(intVal.Int64(), s) } s := bf.Text('g', -1) return createScalarNode(0.0, s) case v.Type().IsTupleType() || v.Type().IsListType() || v.Type().IsSetType(): seq := &CandidateNode{Kind: SequenceNode} it := v.ElementIterator() for it.Next() { _, val := it.Element() seq.AddChild(convertCtyValueToNode(val)) } return seq case v.Type().IsMapType() || v.Type().IsObjectType(): m := &CandidateNode{Kind: MappingNode} it := v.ElementIterator() for it.Next() { key, val := it.Element() keyNode := createStringScalarNode(key.AsString()) valNode := convertCtyValueToNode(val) m.AddKeyValueChild(keyNode, valNode) } return m default: return createStringScalarNode(v.GoString()) } } ================================================ FILE: pkg/yqlib/decoder_ini.go ================================================ //go:build !yq_noini package yqlib import ( "fmt" "io" "github.com/go-ini/ini" ) type iniDecoder struct { reader io.Reader finished bool // Flag to signal completion of processing } func NewINIDecoder() Decoder { return &iniDecoder{ finished: false, // Initialise the flag as false } } func (dec *iniDecoder) Init(reader io.Reader) error { // Store the reader for use in Decode dec.reader = reader return nil } func (dec *iniDecoder) Decode() (*CandidateNode, error) { // If processing is already finished, return io.EOF if dec.finished { return nil, io.EOF } // Read all content from the stored reader content, err := io.ReadAll(dec.reader) if err != nil { return nil, fmt.Errorf("failed to read INI content: %w", err) } // Parse the INI content cfg, err := ini.Load(content) if err != nil { return nil, fmt.Errorf("failed to parse INI content: %w", err) } // Create a root CandidateNode as a MappingNode (since INI is key-value based) root := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Value: "", } // Process each section in the INI file for _, section := range cfg.Sections() { sectionName := section.Name() if sectionName == ini.DefaultSection { // For the default section, add key-value pairs directly to the root node for _, key := range section.Keys() { keyName := key.Name() keyValue := key.String() // Create a key node (scalar for the key name) keyNode := createStringScalarNode(keyName) // Create a value node (scalar for the value) valueNode := createStringScalarNode(keyValue) // Add key-value pair to the root node root.AddKeyValueChild(keyNode, valueNode) } } else { // For named sections, create a nested map sectionNode := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Value: "", } // Add key-value pairs to the section node for _, key := range section.Keys() { keyName := key.Name() keyValue := key.String() // Create a key node (scalar for the key name) keyNode := createStringScalarNode(keyName) // Create a value node (scalar for the value) valueNode := createStringScalarNode(keyValue) // Add key-value pair to the section node sectionNode.AddKeyValueChild(keyNode, valueNode) } // Create a key node for the section name sectionKeyNode := createStringScalarNode(sectionName) // Add the section as a nested map to the root node root.AddKeyValueChild(sectionKeyNode, sectionNode) } } // Set the finished flag to true to prevent further Decode calls dec.finished = true // Return the root node return root, nil } ================================================ FILE: pkg/yqlib/decoder_json.go ================================================ //go:build !yq_nojson package yqlib import ( "io" "github.com/goccy/go-json" ) type jsonDecoder struct { decoder json.Decoder } func NewJSONDecoder() Decoder { return &jsonDecoder{} } func (dec *jsonDecoder) Init(reader io.Reader) error { dec.decoder = *json.NewDecoder(reader) return nil } func (dec *jsonDecoder) Decode() (*CandidateNode, error) { var dataBucket CandidateNode err := dec.decoder.Decode(&dataBucket) if err != nil { return nil, err } return &dataBucket, nil } ================================================ FILE: pkg/yqlib/decoder_lua.go ================================================ //go:build !yq_nolua package yqlib import ( "fmt" "io" "math" lua "github.com/yuin/gopher-lua" ) type luaDecoder struct { reader io.Reader finished bool prefs LuaPreferences } func NewLuaDecoder(prefs LuaPreferences) Decoder { return &luaDecoder{ prefs: prefs, } } func (dec *luaDecoder) Init(reader io.Reader) error { dec.reader = reader return nil } func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *CandidateNode { switch lv.Type() { case lua.LTNil: return &CandidateNode{ Kind: ScalarNode, Tag: "!!null", Value: "", } case lua.LTBool: return &CandidateNode{ Kind: ScalarNode, Tag: "!!bool", Value: lv.String(), } case lua.LTNumber: n := float64(lua.LVAsNumber(lv)) // various special case floats if math.IsNaN(n) { return &CandidateNode{ Kind: ScalarNode, Tag: "!!float", Value: ".nan", } } if math.IsInf(n, 1) { return &CandidateNode{ Kind: ScalarNode, Tag: "!!float", Value: ".inf", } } if math.IsInf(n, -1) { return &CandidateNode{ Kind: ScalarNode, Tag: "!!float", Value: "-.inf", } } // does it look like an integer? if n == float64(int(n)) { return &CandidateNode{ Kind: ScalarNode, Tag: "!!int", Value: lv.String(), } } return &CandidateNode{ Kind: ScalarNode, Tag: "!!float", Value: lv.String(), } case lua.LTString: return &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: lv.String(), } case lua.LTFunction: return &CandidateNode{ Kind: ScalarNode, Tag: "tag:lua.org,2006,function", Value: lv.String(), } case lua.LTTable: // Simultaneously create a sequence and a map, pick which one to return // based on whether all keys were consecutive integers i := 1 yaml_sequence := &CandidateNode{ Kind: SequenceNode, Tag: "!!seq", } yaml_map := &CandidateNode{ Kind: MappingNode, Tag: "!!map", } t := lv.(*lua.LTable) k, v := ls.Next(t, lua.LNil) for k != lua.LNil { if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i { i++ } else { i = 0 } newKey := dec.convertToYamlNode(ls, k) yv := dec.convertToYamlNode(ls, v) yaml_map.AddKeyValueChild(newKey, yv) if i != 0 { yaml_sequence.AddChild(yv) } k, v = ls.Next(t, k) } if i != 0 { return yaml_sequence } return yaml_map default: return &CandidateNode{ Kind: ScalarNode, LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()), Tag: "!!null", Value: lv.String(), } } } func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *CandidateNode { if ls.GetTop() == 0 { // no items were explicitly returned, encode the globals table instead return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex)) } return dec.convertToYamlNode(ls, ls.Get(1)) } func (dec *luaDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } ls := lua.NewState(lua.Options{SkipOpenLibs: true}) defer ls.Close() fn, err := ls.Load(dec.reader, "@input") if err != nil { return nil, err } ls.Push(fn) err = ls.PCall(0, lua.MultRet, nil) if err != nil { return nil, err } firstNode := dec.decideTopLevelNode(ls) dec.finished = true return firstNode, nil } ================================================ FILE: pkg/yqlib/decoder_properties.go ================================================ //go:build !yq_noprops package yqlib import ( "bytes" "fmt" "io" "strconv" "strings" "github.com/magiconair/properties" ) type propertiesDecoder struct { reader io.Reader finished bool d DataTreeNavigator } func NewPropertiesDecoder() Decoder { return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false} } func (dec *propertiesDecoder) Init(reader io.Reader) error { dec.reader = reader dec.finished = false return nil } func parsePropKey(key string) []interface{} { pathStrArray := strings.Split(key, ".") path := make([]interface{}, len(pathStrArray)) for i, pathStr := range pathStrArray { num, err := strconv.ParseInt(pathStr, 10, 32) if err == nil { path[i] = num } else { path[i] = pathStr } } return path } func (dec *propertiesDecoder) processComment(c string) string { if c == "" { return "" } return "# " + c } func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error { assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}} rhsCandidateNode := &CandidateNode{ Tag: "!!str", Value: fmt.Sprintf("%v", path[len(path)-1]), HeadComment: dec.processComment(strings.Join(comments, "\n")), Kind: ScalarNode, } rhsCandidateNode.Tag = rhsCandidateNode.guessTagFromCustomType() rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode} assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, LHS: createTraversalTree(path, traversePreferences{}, true), RHS: &ExpressionNode{Operation: rhsOp}, } _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) return err } func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error { value, _ := properties.Get(key) path := parsePropKey(key) propertyComments := properties.GetComments(key) if len(propertyComments) > 0 { err := dec.applyPropertyComments(context, path, propertyComments) if err != nil { return nil } } rhsNode := createStringScalarNode(value) rhsNode.Tag = rhsNode.guessTagFromCustomType() return dec.d.DeeplyAssign(context, path, rhsNode) } func (dec *propertiesDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } buf := new(bytes.Buffer) if _, err := buf.ReadFrom(dec.reader); err != nil { return nil, err } if buf.Len() == 0 { dec.finished = true return nil, io.EOF } properties, err := properties.LoadString(buf.String()) if err != nil { return nil, err } properties.DisableExpansion = true rootMap := &CandidateNode{ Kind: MappingNode, Tag: "!!map", } context := Context{} context = context.SingleChildContext(rootMap) for _, key := range properties.Keys() { if err := dec.applyProperty(context, properties, key); err != nil { return nil, err } } dec.finished = true return rootMap, nil } ================================================ FILE: pkg/yqlib/decoder_test.go ================================================ package yqlib import ( "bufio" "bytes" "fmt" "strings" ) type formatScenario struct { input string indent int expression string expected string description string subdescription string skipDoc bool scenarioType string expectedError string } func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) { var output bytes.Buffer writer := bufio.NewWriter(&output) if decoder == nil { decoder = NewYamlDecoder(ConfiguredYamlPreferences) } log.Debugf("reading docs") inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder) if err != nil { return "", err } log.Debugf("done reading the documents") expression := s.expression if expression == "" { expression = "." } exp, err := getExpressionParser().ParseExpression(expression) if err != nil { return "", err } context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp) log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes)) if err != nil { return "", err } printer := NewPrinter(encoder, NewSinglePrinterWriter(writer)) err = printer.PrintResults(context.MatchingNodes) if err != nil { return "", err } writer.Flush() return output.String(), nil } func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string { result, err := processFormatScenario(s, decoder, encoder) if err != nil { log.Error("Bad scenario %v: %w", s.description, err) return fmt.Sprintf("Bad scenario %v: %v", s.description, err.Error()) } return result } ================================================ FILE: pkg/yqlib/decoder_toml.go ================================================ //go:build !yq_notoml package yqlib import ( "bytes" "errors" "fmt" "io" "strconv" "strings" "time" toml "github.com/pelletier/go-toml/v2/unstable" ) type tomlDecoder struct { parser toml.Parser finished bool d DataTreeNavigator rootMap *CandidateNode pendingComments []string // Head comments collected from Comment nodes firstContentSeen bool // Track if we've processed the first non-comment node } func NewTomlDecoder() Decoder { return &tomlDecoder{ finished: false, d: NewDataTreeNavigator(), } } func (dec *tomlDecoder) Init(reader io.Reader) error { dec.parser = toml.Parser{KeepComments: true} buf := new(bytes.Buffer) _, err := buf.ReadFrom(reader) if err != nil { return err } dec.parser.Reset(buf.Bytes()) dec.rootMap = &CandidateNode{ Kind: MappingNode, Tag: "!!map", } dec.pendingComments = make([]string, 0) dec.firstContentSeen = false return nil } func (dec *tomlDecoder) attachOrphanedCommentsToNode(tableNodeValue *CandidateNode) { if len(dec.pendingComments) > 0 { comments := strings.Join(dec.pendingComments, "\n") if tableNodeValue.HeadComment == "" { tableNodeValue.HeadComment = comments } else { tableNodeValue.HeadComment = tableNodeValue.HeadComment + "\n" + comments } dec.pendingComments = make([]string, 0) } } func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} { path := make([]interface{}, 0) for { path = append(path, string(tomlNode.Data)) tomlNode = tomlNode.Next() if tomlNode == nil { return path } } } func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error { value := tomlNode.Value() path := dec.getFullPath(value.Next()) valueNode, err := dec.decodeNode(value) if err != nil { return err } // Attach pending head comments if len(dec.pendingComments) > 0 { valueNode.HeadComment = strings.Join(dec.pendingComments, "\n") dec.pendingComments = make([]string, 0) } // Check for inline comment chained to the KeyValue node nextNode := tomlNode.Next() if nextNode != nil && nextNode.Kind == toml.Comment { valueNode.LineComment = string(nextNode.Data) } context := Context{} context = context.SingleChildContext(rootMap) return dec.d.DeeplyAssign(context, path, valueNode) } func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) { log.Debug("decodeKeyValuesIntoMap -- processing first (current) entry") if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil { return false, err } for dec.parser.NextExpression() { nextItem := dec.parser.Expression() log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind) switch nextItem.Kind { case toml.KeyValue: if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil { return false, err } case toml.Comment: // Standalone comment - add to pending for next element dec.pendingComments = append(dec.pendingComments, string(nextItem.Data)) default: // run out of key values log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind) return true, nil } } log.Debug("no more things to read in") return false, nil } func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) { content := make([]*CandidateNode, 0) log.Debug("createInlineTableMap") iterator := tomlNode.Children() for iterator.Next() { child := iterator.Node() if child.Kind != toml.KeyValue { return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind) } keyValues := &CandidateNode{ Kind: MappingNode, Tag: "!!map", } if err := dec.processKeyValueIntoMap(keyValues, child); err != nil { return nil, err } content = append(content, keyValues.Content...) } return &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: content, }, nil } func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) { content := make([]*CandidateNode, 0) var pendingArrayComments []string iterator := tomlNode.Children() for iterator.Next() { child := iterator.Node() // Handle comments within arrays if child.Kind == toml.Comment { // Collect comments to attach to the next array element pendingArrayComments = append(pendingArrayComments, string(child.Data)) continue } yamlNode, err := dec.decodeNode(child) if err != nil { return nil, err } // Attach any pending comments to this array element if len(pendingArrayComments) > 0 { yamlNode.HeadComment = strings.Join(pendingArrayComments, "\n") pendingArrayComments = make([]string, 0) } content = append(content, yamlNode) } return &CandidateNode{ Kind: SequenceNode, Tag: "!!seq", Content: content, }, nil } func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) return createScalarNode(content, content), nil } func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) return createScalarNode(content == "true", content), nil } func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) _, num, err := parseInt64(content) return createScalarNode(num, content), err } func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) val, err := parseDateTime(time.RFC3339, content) return createScalarNode(val, content), err } func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) { content := string(tomlNode.Data) num, err := strconv.ParseFloat(content, 64) return createScalarNode(num, content), err } func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) { switch tomlNode.Kind { case toml.Key, toml.String: return dec.createStringScalar(tomlNode) case toml.Bool: return dec.createBoolScalar(tomlNode) case toml.Integer: return dec.createIntegerScalar(tomlNode) case toml.DateTime: return dec.createDateTimeScalar(tomlNode) case toml.Float: return dec.createFloatScalar(tomlNode) case toml.Array: return dec.createArray(tomlNode) case toml.InlineTable: return dec.createInlineTableMap(tomlNode) default: return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind) } } func (dec *tomlDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } // // toml library likes to panic var deferredError error defer func() { //catch or finally if r := recover(); r != nil { var ok bool deferredError, ok = r.(error) if !ok { deferredError = fmt.Errorf("pkg: %v", r) } } }() log.Debug("ok here we go") var runAgainstCurrentExp = false var err error for runAgainstCurrentExp || dec.parser.NextExpression() { if runAgainstCurrentExp { log.Debug("running against current exp") } currentNode := dec.parser.Expression() log.Debug("currentNode: %v ", currentNode.Kind) runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode) if err != nil { return dec.rootMap, err } } err = dec.parser.Error() if err != nil { return nil, err } // must have finished dec.finished = true if len(dec.rootMap.Content) == 0 { return nil, io.EOF } return dec.rootMap, deferredError } func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) { var runAgainstCurrentExp bool var err error log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap)) switch currentNode.Kind { case toml.Comment: // Collect comment to attach to next element commentText := string(currentNode.Data) // If we haven't seen any content yet, accumulate comments for root if !dec.firstContentSeen { if dec.rootMap.HeadComment == "" { dec.rootMap.HeadComment = commentText } else { dec.rootMap.HeadComment = dec.rootMap.HeadComment + "\n" + commentText } } else { // We've seen content, so these comments are for the next element dec.pendingComments = append(dec.pendingComments, commentText) } return false, nil case toml.Table: dec.firstContentSeen = true runAgainstCurrentExp, err = dec.processTable(currentNode) case toml.ArrayTable: dec.firstContentSeen = true runAgainstCurrentExp, err = dec.processArrayTable(currentNode) default: dec.firstContentSeen = true runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode) } log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap)) return runAgainstCurrentExp, err } func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) { log.Debug("Enter processTable") child := currentNode.Child() fullPath := dec.getFullPath(child) log.Debug("fullpath: %v", fullPath) c := Context{} c = c.SingleChildContext(dec.rootMap) fullPath, err := getPathToUse(fullPath, dec, c) if err != nil { return false, err } tableNodeValue := &CandidateNode{ Kind: MappingNode, Tag: "!!map", Content: make([]*CandidateNode, 0), EncodeSeparate: true, } // Attach pending head comments to the table if len(dec.pendingComments) > 0 { tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n") dec.pendingComments = make([]string, 0) } var tableValue *toml.Node runAgainstCurrentExp := false sawKeyValue := false for dec.parser.NextExpression() { tableValue = dec.parser.Expression() // Allow standalone comments inside the table before the first key-value. // These should be associated with the next element in the table (usually the first key-value), // not treated as "end of table" (which would cause subsequent key-values to be parsed at root). if tableValue.Kind == toml.Comment { dec.pendingComments = append(dec.pendingComments, string(tableValue.Data)) continue } // next expression is not table data, so we are done (but we need to re-process it at top-level) if tableValue.Kind != toml.KeyValue { log.Debug("got an empty table (or reached next section)") // If the table had only comments, attach them to the table itself so they don't leak to the next node. if !sawKeyValue { dec.attachOrphanedCommentsToNode(tableNodeValue) } runAgainstCurrentExp = true break } sawKeyValue = true runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue) if err != nil && !errors.Is(err, io.EOF) { return false, err } break } // If we hit EOF after only seeing comments inside this table, attach them to the table itself // so they don't leak to whatever comes next. if !sawKeyValue { dec.attachOrphanedCommentsToNode(tableNodeValue) } err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue) if err != nil { return false, err } return runAgainstCurrentExp, nil } func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error { log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode)) rhsCandidateNode := &CandidateNode{ Kind: SequenceNode, Tag: "!!seq", Content: []*CandidateNode{rhsNode}, } assignmentOp := &Operation{OperationType: addAssignOpType} rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode} assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, LHS: createTraversalTree(path, traversePreferences{}, false), RHS: &ExpressionNode{Operation: rhsOp}, } _, err := dec.d.GetMatchingNodes(context, assignmentOpNode) return err } func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) { log.Debug("Enter processArrayTable") child := currentNode.Child() fullPath := dec.getFullPath(child) log.Debug("Fullpath: %v", fullPath) c := Context{} c = c.SingleChildContext(dec.rootMap) fullPath, err := getPathToUse(fullPath, dec, c) if err != nil { return false, err } // need to use the array append exp to add another entry to // this array: fullpath += [ thing ] hasValue := dec.parser.NextExpression() tableNodeValue := &CandidateNode{ Kind: MappingNode, Tag: "!!map", EncodeSeparate: true, } // Attach pending head comments to the array table if len(dec.pendingComments) > 0 { tableNodeValue.HeadComment = strings.Join(dec.pendingComments, "\n") dec.pendingComments = make([]string, 0) } runAgainstCurrentExp := false sawKeyValue := false if hasValue { for { exp := dec.parser.Expression() // Allow standalone comments inside array tables before the first key-value. if exp.Kind == toml.Comment { dec.pendingComments = append(dec.pendingComments, string(exp.Data)) hasValue = dec.parser.NextExpression() if !hasValue { break } continue } // if the next value is a ArrayTable or Table, then its not part of this declaration (not a key value pair) // so lets leave that expression for the next round of parsing if exp.Kind == toml.ArrayTable || exp.Kind == toml.Table { // If this array-table entry had only comments, attach them to the entry so they don't leak. if !sawKeyValue { dec.attachOrphanedCommentsToNode(tableNodeValue) } runAgainstCurrentExp = true break } sawKeyValue = true // otherwise, if there is a value, it must be some key value pairs of the // first object in the array! runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, exp) if err != nil && !errors.Is(err, io.EOF) { return false, err } break } } // If we hit EOF after only seeing comments inside this array-table entry, attach them to the entry // so they don't leak to whatever comes next. if !sawKeyValue && len(dec.pendingComments) > 0 { comments := strings.Join(dec.pendingComments, "\n") if tableNodeValue.HeadComment == "" { tableNodeValue.HeadComment = comments } else { tableNodeValue.HeadComment = tableNodeValue.HeadComment + "\n" + comments } dec.pendingComments = make([]string, 0) } // += function err = dec.arrayAppend(c, fullPath, tableNodeValue) return runAgainstCurrentExp, err } // if fullPath points to an array of maps rather than a map // then it should set this element into the _last_ element of that array. // Because TOML. So we'll inject the last index into the path. func getPathToUse(fullPath []interface{}, dec *tomlDecoder, c Context) ([]interface{}, error) { // We need to check the entire path (except the last element), not just the immediate parent, // because we may have nested array tables like [[array.subarray.subsubarray]] // where both 'array' and 'subarray' are arrays that already exist. if len(fullPath) == 0 { return fullPath, nil } resultPath := make([]interface{}, 0, len(fullPath)*2) // preallocate with extra space for indices // Process all segments except the last one for i := 0; i < len(fullPath)-1; i++ { resultPath = append(resultPath, fullPath[i]) // Check if the current path segment points to an array readOp := createTraversalTree(resultPath, traversePreferences{DontAutoCreate: true}, false) resultContext, err := dec.d.GetMatchingNodes(c, readOp) if err != nil { return nil, err } if resultContext.MatchingNodes.Len() >= 1 { match := resultContext.MatchingNodes.Front().Value.(*CandidateNode) // If this segment points to an array, we need to add the last index // before continuing with the rest of the path if match.Kind == SequenceNode && len(match.Content) > 0 { lastIndex := len(match.Content) - 1 resultPath = append(resultPath, lastIndex) log.Debugf("Path segment %v is an array, injecting index %d", resultPath[:len(resultPath)-1], lastIndex) } } } // Add the last segment resultPath = append(resultPath, fullPath[len(fullPath)-1]) log.Debugf("getPathToUse: original path %v -> result path %v", fullPath, resultPath) return resultPath, nil } ================================================ FILE: pkg/yqlib/decoder_uri.go ================================================ //go:build !yq_nouri package yqlib import ( "bytes" "io" "net/url" ) type uriDecoder struct { reader io.Reader finished bool readAnything bool } func NewUriDecoder() Decoder { return &uriDecoder{finished: false} } func (dec *uriDecoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false return nil } func (dec *uriDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } buf := new(bytes.Buffer) if _, err := buf.ReadFrom(dec.reader); err != nil { return nil, err } if buf.Len() == 0 { dec.finished = true // if we've read _only_ an empty string, lets return that // otherwise if we've already read some bytes, and now we get // an empty string, then we are done. if dec.readAnything { return nil, io.EOF } } newValue, err := url.QueryUnescape(buf.String()) if err != nil { return nil, err } dec.readAnything = true return createStringScalarNode(newValue), nil } ================================================ FILE: pkg/yqlib/decoder_uri_test.go ================================================ //go:build !yq_nouri package yqlib import ( "io" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) func TestUriDecoder_Init(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("test") err := decoder.Init(reader) test.AssertResult(t, nil, err) } func TestUriDecoder_DecodeSimpleString(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("hello%20world") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "!!str", node.Tag) test.AssertResult(t, "hello world", node.Value) } func TestUriDecoder_DecodeSpecialCharacters(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("hello%21%40%23%24%25") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "hello!@#$%", node.Value) } func TestUriDecoder_DecodeUTF8(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("%E2%9C%93%20check") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "✓ check", node.Value) } func TestUriDecoder_DecodePlusSign(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("a+b") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) // Note: url.QueryUnescape does NOT convert + to space // That's only for form encoding (url.ParseQuery) test.AssertResult(t, "a b", node.Value) } func TestUriDecoder_DecodeEmptyString(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "", node.Value) // Second decode should return EOF node, err = decoder.Decode() test.AssertResult(t, io.EOF, err) test.AssertResult(t, (*CandidateNode)(nil), node) } func TestUriDecoder_DecodeMultipleCalls(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("test") err := decoder.Init(reader) test.AssertResult(t, nil, err) // First decode node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "test", node.Value) // Second decode should return EOF since we've consumed all input node, err = decoder.Decode() test.AssertResult(t, io.EOF, err) test.AssertResult(t, (*CandidateNode)(nil), node) } func TestUriDecoder_DecodeInvalidEscape(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("test%ZZ") err := decoder.Init(reader) test.AssertResult(t, nil, err) _, err = decoder.Decode() // Should return an error for invalid escape sequence if err == nil { t.Error("Expected error for invalid escape sequence, got nil") } } func TestUriDecoder_DecodeSlashAndQuery(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("path%2Fto%2Ffile%3Fquery%3Dvalue") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "path/to/file?query=value", node.Value) } func TestUriDecoder_DecodePercent(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("100%25") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "100%", node.Value) } func TestUriDecoder_DecodeNoEscaping(t *testing.T) { decoder := NewUriDecoder() reader := strings.NewReader("simple_text-123") err := decoder.Init(reader) test.AssertResult(t, nil, err) node, err := decoder.Decode() test.AssertResult(t, nil, err) test.AssertResult(t, "simple_text-123", node.Value) } // Mock reader that returns an error type errorReader struct{} func (e *errorReader) Read(_ []byte) (n int, err error) { return 0, io.ErrUnexpectedEOF } func TestUriDecoder_DecodeReadError(t *testing.T) { decoder := NewUriDecoder() err := decoder.Init(&errorReader{}) test.AssertResult(t, nil, err) _, err = decoder.Decode() test.AssertResult(t, io.ErrUnexpectedEOF, err) } ================================================ FILE: pkg/yqlib/decoder_xml.go ================================================ //go:build !yq_noxml package yqlib import ( "encoding/xml" "errors" "fmt" "io" "regexp" "strings" "unicode" "golang.org/x/net/html/charset" ) type xmlDecoder struct { reader io.Reader readAnything bool finished bool prefs XmlPreferences } func NewXMLDecoder(prefs XmlPreferences) Decoder { return &xmlDecoder{ finished: false, prefs: prefs, } } func (dec *xmlDecoder) Init(reader io.Reader) error { dec.reader = reader dec.readAnything = false dec.finished = false return nil } func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) { yamlNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} for _, child := range nodes { yamlChild, err := dec.convertToYamlNode(child) if err != nil { return nil, err } yamlNode.AddChild(yamlChild) } return yamlNode, nil } var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`) func (dec *xmlDecoder) processComment(c string) string { if c == "" { return "" } //need to replace "cat " with "# cat" // "\ncat\n" with "\n cat\n" // ensure non-empty comments starting with newline have a space in front replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2") replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#") return replacement } func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) { log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment) yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"} if len(n.Data) > 0 { log.Debugf("creating content node for map: %v", dec.prefs.ContentName) label := dec.prefs.ContentName labelNode := createScalarNode(label, label) labelNode.HeadComment = dec.processComment(n.HeadComment) labelNode.LineComment = dec.processComment(n.LineComment) labelNode.FootComment = dec.processComment(n.FootComment) yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data)) } for i, keyValuePair := range n.Children { label := keyValuePair.K children := keyValuePair.V labelNode := createScalarNode(label, label) var valueNode *CandidateNode var err error if i == 0 { log.Debugf("head comment here") labelNode.HeadComment = dec.processComment(n.HeadComment) } log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment) labelNode.FootComment = dec.processComment(keyValuePair.FootComment) log.Debug("len of children in %v is %v", label, len(children)) if len(children) > 1 { valueNode, err = dec.createSequence(children) if err != nil { return nil, err } } else { // comment hack for maps of scalars // if the value is a scalar, the head comment of the scalar needs to go on the key? // add tests for as well as multiple of inputXmlWithComments > yaml if len(children[0].Children) == 0 && children[0].HeadComment != "" { if len(children[0].Data) > 0 { log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment) labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n") children[0].HeadComment = "" } else { // child is null, put the headComment as a linecomment for reasons children[0].LineComment = children[0].HeadComment children[0].HeadComment = "" } } valueNode, err = dec.convertToYamlNode(children[0]) if err != nil { return nil, err } } yamlNode.AddKeyValueChild(labelNode, valueNode) } return yamlNode, nil } func (dec *xmlDecoder) createValueNodeFromData(values []string) *CandidateNode { switch len(values) { case 0: return createScalarNode(nil, "") case 1: return createScalarNode(values[0], values[0]) default: content := make([]*CandidateNode, 0) for _, value := range values { content = append(content, createScalarNode(value, value)) } return &CandidateNode{ Kind: SequenceNode, Tag: "!!seq", Content: content, } } } func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) { if len(n.Children) > 0 { return dec.createMap(n) } scalar := dec.createValueNodeFromData(n.Data) log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment) scalar.HeadComment = dec.processComment(n.HeadComment) scalar.LineComment = dec.processComment(n.LineComment) if scalar.Tag == "!!seq" { scalar.Content[0].HeadComment = scalar.LineComment scalar.LineComment = "" } scalar.FootComment = dec.processComment(n.FootComment) return scalar, nil } func (dec *xmlDecoder) Decode() (*CandidateNode, error) { if dec.finished { return nil, io.EOF } root := &xmlNode{} // cant use xj - it doesn't keep map order. err := dec.decodeXML(root) if err != nil { return nil, err } firstNode, err := dec.convertToYamlNode(root) if err != nil { return nil, err } else if firstNode.Tag == "!!null" { dec.finished = true if dec.readAnything { return nil, io.EOF } } dec.readAnything = true dec.finished = true return firstNode, nil } type xmlNode struct { Children []*xmlChildrenKv HeadComment string FootComment string LineComment string Data []string } type xmlChildrenKv struct { K string V []*xmlNode FootComment string } // AddChild appends a node to the list of children func (n *xmlNode) AddChild(s string, c *xmlNode) { if n.Children == nil { n.Children = make([]*xmlChildrenKv, 0) } log.Debug("looking for %s", s) // see if we can find an existing entry to add to for _, childEntry := range n.Children { if childEntry.K == s { log.Debug("found it, appending an entry%s", s) childEntry.V = append(childEntry.V, c) log.Debug("yay len of children in %v is %v", s, len(childEntry.V)) return } } log.Debug("not there, making a new one %s", s) n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}}) } type element struct { parent *element n *xmlNode label string state string } // this code is heavily based on https://github.com/basgys/goxml2json // main changes are to decode into a structure that preserves the original order // of the map keys. func (dec *xmlDecoder) decodeXML(root *xmlNode) error { xmlDec := xml.NewDecoder(dec.reader) xmlDec.Strict = dec.prefs.StrictMode // That will convert the charset if the provided XML is non-UTF-8 xmlDec.CharsetReader = charset.NewReaderLabel started := false // Create first element from the root node elem := &element{ parent: nil, n: root, } getToken := func() (xml.Token, error) { if dec.prefs.UseRawToken { return xmlDec.RawToken() } return xmlDec.Token() } for { t, e := getToken() if e != nil && !errors.Is(e, io.EOF) { return e } if t == nil { break } switch se := t.(type) { case xml.StartElement: log.Debug("start element %v", se.Name.Local) elem.state = "started" // Build new a new current element and link it to its parent var label = se.Name.Local if dec.prefs.KeepNamespace { if se.Name.Space != "" { label = se.Name.Space + ":" + se.Name.Local } } elem = &element{ parent: elem, n: &xmlNode{}, label: label, } // Extract attributes as children for _, a := range se.Attr { if dec.prefs.KeepNamespace { if a.Name.Space != "" { a.Name.Local = a.Name.Space + ":" + a.Name.Local } } elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: []string{a.Value}}) } case xml.CharData: // Extract XML data (if any) newBit := trimNonGraphic(string(se)) if !started && len(newBit) > 0 { return fmt.Errorf("invalid XML: Encountered chardata [%v] outside of XML node", newBit) } if len(newBit) > 0 { elem.n.Data = append(elem.n.Data, newBit) elem.state = "chardata" log.Debug("chardata [%v] for %v", elem.n.Data, elem.label) } case xml.EndElement: if elem == nil { log.Debug("no element, probably bad xml") continue } log.Debug("end element %v", elem.label) elem.state = "finished" // And add it to its parent list if elem.parent != nil { elem.parent.n.AddChild(elem.label, elem.n) } // Then change the current element to its parent elem = elem.parent case xml.Comment: commentStr := string(xml.CharData(se)) switch elem.state { case "started": applyFootComment(elem, commentStr) case "chardata": log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ") default: log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr) elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ") } case xml.ProcInst: if !dec.prefs.SkipProcInst { elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: []string{string(se.Inst)}}) } case xml.Directive: if !dec.prefs.SkipDirectives { elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: []string{string(se)}}) } } started = true } return nil } func applyFootComment(elem *element, commentStr string) { // first lets try to put the comment on the last child if len(elem.n.Children) > 0 { lastChildIndex := len(elem.n.Children) - 1 childKv := elem.n.Children[lastChildIndex] log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr) // if it's an array of scalars, put the foot comment on the scalar itself if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 { nodeToUpdate := childKv.V[len(childKv.V)-1] nodeToUpdate.FootComment = joinComments([]string{nodeToUpdate.FootComment, commentStr}, " ") } else { childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ") } } else { log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr) elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ") } } func joinComments(rawStrings []string, joinStr string) string { stringsToJoin := make([]string, 0) for _, str := range rawStrings { if str != "" { stringsToJoin = append(stringsToJoin, str) } } return strings.Join(stringsToJoin, joinStr) } // trimNonGraphic returns a slice of the string s, with all leading and trailing // non graphic characters and spaces removed. // // Graphic characters include letters, marks, numbers, punctuation, symbols, // and spaces, from categories L, M, N, P, S, Zs. // Spacing characters are set by category Z and property Pattern_White_Space. func trimNonGraphic(s string) string { if s == "" { return s } var first *int var last int for i, r := range []rune(s) { if !unicode.IsGraphic(r) || unicode.IsSpace(r) { continue } if first == nil { f := i // copy i first = &f last = i } else { last = i } } // If first is nil, it means there are no graphic characters if first == nil { return "" } return string([]rune(s)[*first : last+1]) } ================================================ FILE: pkg/yqlib/decoder_yaml.go ================================================ package yqlib import ( "bufio" "bytes" "errors" "io" "regexp" "strings" yaml "go.yaml.in/yaml/v4" ) var ( commentLineRe = regexp.MustCompile(`^\s*#`) yamlDirectiveLineRe = regexp.MustCompile(`^\s*%YAML`) separatorLineRe = regexp.MustCompile(`^\s*---\s*$`) separatorPrefixRe = regexp.MustCompile(`^\s*---\s+`) ) type yamlDecoder struct { decoder yaml.Decoder prefs YamlPreferences // work around of various parsing issues by yaml.v3 with document headers leadingContent string bufferRead bytes.Buffer // anchor map persists over multiple documents for convenience. anchorMap map[string]*CandidateNode readAnything bool firstFile bool documentIndex uint } func NewYamlDecoder(prefs YamlPreferences) Decoder { return &yamlDecoder{prefs: prefs, firstFile: true} } func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) { var sb strings.Builder for { line, err := reader.ReadString('\n') if errors.Is(err, io.EOF) && line == "" { // no more data return reader, sb.String(), nil } if err != nil && !errors.Is(err, io.EOF) { return reader, sb.String(), err } // Determine newline style and strip it for inspection newline := "" if strings.HasSuffix(line, "\r\n") { newline = "\r\n" line = strings.TrimSuffix(line, "\r\n") } else if strings.HasSuffix(line, "\n") { newline = "\n" line = strings.TrimSuffix(line, "\n") } trimmed := strings.TrimSpace(line) // Document separator: exact line '---' or a '--- ' prefix followed by content if separatorLineRe.MatchString(trimmed) { sb.WriteString("$yqDocSeparator$") sb.WriteString(newline) if errors.Is(err, io.EOF) { return reader, sb.String(), nil } continue } // Handle lines that start with '--- ' followed by more content (e.g. '--- cat') if separatorPrefixRe.MatchString(line) { match := separatorPrefixRe.FindString(line) remainder := line[len(match):] // normalise separator newline: if original had none, default to LF sepNewline := newline if sepNewline == "" { sepNewline = "\n" } sb.WriteString("$yqDocSeparator$") sb.WriteString(sepNewline) // push the remainder back onto the reader and continue processing reader = bufio.NewReader(io.MultiReader(strings.NewReader(remainder), reader)) if errors.Is(err, io.EOF) && remainder == "" { return reader, sb.String(), nil } continue } // Comments, YAML directives, and blank lines are leading content if commentLineRe.MatchString(line) || yamlDirectiveLineRe.MatchString(line) || trimmed == "" { sb.WriteString(line) sb.WriteString(newline) if errors.Is(err, io.EOF) { return reader, sb.String(), nil } continue } // First non-leading line: push it back onto a reader and return originalLine := line + newline return io.MultiReader(strings.NewReader(originalLine), reader), sb.String(), nil } } func (dec *yamlDecoder) Init(reader io.Reader) error { readerToUse := reader leadingContent := "" dec.bufferRead = bytes.Buffer{} var err error // if we 'evaluating together' - we only process the leading content // of the first file - this ensures comments from subsequent files are // merged together correctly. if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) { readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader)) if err != nil { return err } } else if !dec.prefs.LeadingContentPreProcessing { // if we're not process the leading content // keep a copy of what we've read. This is incase its a // doc with only comments - the decoder will return nothing // then we can read the comments from bufferRead readerToUse = io.TeeReader(reader, &dec.bufferRead) } dec.leadingContent = leadingContent dec.readAnything = false dec.decoder = *yaml.NewDecoder(readerToUse) dec.firstFile = false dec.documentIndex = 0 dec.anchorMap = make(map[string]*CandidateNode) return nil } func (dec *yamlDecoder) Decode() (*CandidateNode, error) { var yamlNode yaml.Node err := dec.decoder.Decode(&yamlNode) if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything { // force returning an empty node with a comment. dec.readAnything = true return dec.blankNodeWithComment(), nil } else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything { // didn't find any yaml, // check the tee buffer, maybe there were comments dec.readAnything = true dec.leadingContent = dec.bufferRead.String() if dec.leadingContent != "" { return dec.blankNodeWithComment(), nil } return nil, err } else if err != nil { return nil, err } else if len(yamlNode.Content) == 0 { return nil, errors.New("yaml node has no content") } candidateNode := CandidateNode{document: dec.documentIndex} // don't bother with the DocumentNode err = candidateNode.UnmarshalYAML(yamlNode.Content[0], dec.anchorMap) if err != nil { return nil, err } candidateNode.HeadComment = yamlNode.HeadComment + candidateNode.HeadComment candidateNode.FootComment = yamlNode.FootComment + candidateNode.FootComment if dec.leadingContent != "" { candidateNode.LeadingContent = dec.leadingContent dec.leadingContent = "" } dec.readAnything = true dec.documentIndex++ return &candidateNode, nil } func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode { node := createScalarNode(nil, "") node.LeadingContent = dec.leadingContent return node } ================================================ FILE: pkg/yqlib/doc/.gitignore ================================================ *.zip ================================================ FILE: pkg/yqlib/doc/notification-snippet.md ================================================ ================================================ FILE: pkg/yqlib/doc/operators/add.md ================================================ # Add Add behaves differently according to the type of the LHS: * arrays: concatenate * number scalars: arithmetic addition * string scalars: concatenate * maps: shallow merge (use the multiply operator (`*`) to deeply merge) Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. ## Concatenate arrays Given a sample.yml file of: ```yaml a: - 1 - 2 b: - 3 - 4 ``` then ```bash yq '.a + .b' sample.yml ``` will output ```yaml - 1 - 2 - 3 - 4 ``` ## Concatenate to existing array Note that the styling of `a` is kept. Given a sample.yml file of: ```yaml a: [1,2] b: - 3 - 4 ``` then ```bash yq '.a += .b' sample.yml ``` will output ```yaml a: [1, 2, 3, 4] b: - 3 - 4 ``` ## Concatenate null to array Given a sample.yml file of: ```yaml a: - 1 - 2 ``` then ```bash yq '.a + null' sample.yml ``` will output ```yaml - 1 - 2 ``` ## Append to existing array Note that the styling is copied from existing array elements Given a sample.yml file of: ```yaml a: ['dog'] ``` then ```bash yq '.a += "cat"' sample.yml ``` will output ```yaml a: ['dog', 'cat'] ``` ## Prepend to existing array Given a sample.yml file of: ```yaml a: - dog ``` then ```bash yq '.a = ["cat"] + .a' sample.yml ``` will output ```yaml a: - cat - dog ``` ## Add new object to array Given a sample.yml file of: ```yaml a: - dog: woof ``` then ```bash yq '.a + {"cat": "meow"}' sample.yml ``` will output ```yaml - dog: woof - cat: meow ``` ## Relative append Given a sample.yml file of: ```yaml a: a1: b: - cat a2: b: - dog a3: {} ``` then ```bash yq '.a[].b += ["mouse"]' sample.yml ``` will output ```yaml a: a1: b: - cat - mouse a2: b: - dog - mouse a3: b: - mouse ``` ## String concatenation Given a sample.yml file of: ```yaml a: cat b: meow ``` then ```bash yq '.a += .b' sample.yml ``` will output ```yaml a: catmeow b: meow ``` ## Number addition - float If the lhs or rhs are floats then the expression will be calculated with floats. Given a sample.yml file of: ```yaml a: 3 b: 4.9 ``` then ```bash yq '.a = .a + .b' sample.yml ``` will output ```yaml a: 7.9 b: 4.9 ``` ## Number addition - int If both the lhs and rhs are ints then the expression will be calculated with ints. Given a sample.yml file of: ```yaml a: 3 b: 4 ``` then ```bash yq '.a = .a + .b' sample.yml ``` will output ```yaml a: 7 b: 4 ``` ## Increment numbers Given a sample.yml file of: ```yaml a: 3 b: 5 ``` then ```bash yq '.[] += 1' sample.yml ``` will output ```yaml a: 4 b: 6 ``` ## Date addition You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. Given a sample.yml file of: ```yaml a: 2021-01-01T00:00:00Z ``` then ```bash yq '.a += "3h10m"' sample.yml ``` will output ```yaml a: 2021-01-01T03:10:00Z ``` ## Date addition - custom format You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 6:00AM GMT ``` ## Add to null Adding to null simply returns the rhs Running ```bash yq --null-input 'null + "cat"' ``` will output ```yaml cat ``` ## Add maps to shallow merge Adding objects together shallow merges them. Use `*` to deeply merge. Given a sample.yml file of: ```yaml a: thing: name: Astuff value: x a1: cool b: thing: name: Bstuff legs: 3 b1: neat ``` then ```bash yq '.a += .b' sample.yml ``` will output ```yaml a: thing: name: Bstuff legs: 3 a1: cool b1: neat b: thing: name: Bstuff legs: 3 b1: neat ``` ## Custom types: that are really strings When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml a: !horse cat b: !goat _meow ``` then ```bash yq '.a += .b' sample.yml ``` will output ```yaml a: !horse cat_meow b: !goat _meow ``` ## Custom types: that are really numbers When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml a: !horse 1.2 b: !goat 2.3 ``` then ```bash yq '.a += .b' sample.yml ``` will output ```yaml a: !horse 3.5 b: !goat 2.3 ``` ================================================ FILE: pkg/yqlib/doc/operators/alternative-default-value.md ================================================ # Alternative (Default value) This operator is used to provide alternative (or default) values when a particular expression is either null or false. ## LHS is defined Given a sample.yml file of: ```yaml a: bridge ``` then ```bash yq '.a // "hello"' sample.yml ``` will output ```yaml bridge ``` ## LHS is not defined Given a sample.yml file of: ```yaml {} ``` then ```bash yq '.a // "hello"' sample.yml ``` will output ```yaml hello ``` ## LHS is null Given a sample.yml file of: ```yaml a: ~ ``` then ```bash yq '.a // "hello"' sample.yml ``` will output ```yaml hello ``` ## LHS is false Given a sample.yml file of: ```yaml a: false ``` then ```bash yq '.a // "hello"' sample.yml ``` will output ```yaml hello ``` ## RHS is an expression Given a sample.yml file of: ```yaml a: false b: cat ``` then ```bash yq '.a // .b' sample.yml ``` will output ```yaml cat ``` ## Update or create - entity exists This initialises `a` if it's not present Given a sample.yml file of: ```yaml a: 1 ``` then ```bash yq '(.a // (.a = 0)) += 1' sample.yml ``` will output ```yaml a: 2 ``` ## Update or create - entity does not exist This initialises `a` if it's not present Given a sample.yml file of: ```yaml b: camel ``` then ```bash yq '(.a // (.a = 0)) += 1' sample.yml ``` will output ```yaml b: camel a: 1 ``` ================================================ FILE: pkg/yqlib/doc/operators/anchor-and-alias-operators.md ================================================ # Anchor and Alias Operators Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names). `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed). This flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed. Long story short, you should be setting this flag to true. See examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on. ## Merge one map see https://yaml.org/type/merge.html Given a sample.yml file of: ```yaml - &CENTRE x: 1 y: 2 - &LEFT x: 0 y: 2 - &BIG r: 10 - &SMALL r: 1 - !!merge <<: *CENTRE r: 10 ``` then ```bash yq '.[4] | explode(.)' sample.yml ``` will output ```yaml x: 1 y: 2 r: 10 ``` ## Get anchor Given a sample.yml file of: ```yaml a: &billyBob cat ``` then ```bash yq '.a | anchor' sample.yml ``` will output ```yaml billyBob ``` ## Set anchor Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a anchor = "foobar"' sample.yml ``` will output ```yaml a: &foobar cat ``` ## Set anchor relatively using assign-update Given a sample.yml file of: ```yaml a: b: cat ``` then ```bash yq '.a anchor |= .b' sample.yml ``` will output ```yaml a: &cat b: cat ``` ## Get alias Given a sample.yml file of: ```yaml b: &billyBob meow a: *billyBob ``` then ```bash yq '.a | alias' sample.yml ``` will output ```yaml billyBob ``` ## Set alias Given a sample.yml file of: ```yaml b: &meow purr a: cat ``` then ```bash yq '.a alias = "meow"' sample.yml ``` will output ```yaml b: &meow purr a: *meow ``` ## Set alias to blank does nothing Given a sample.yml file of: ```yaml b: &meow purr a: cat ``` then ```bash yq '.a alias = ""' sample.yml ``` will output ```yaml b: &meow purr a: cat ``` ## Set alias relatively using assign-update Given a sample.yml file of: ```yaml b: &meow purr a: f: meow ``` then ```bash yq '.a alias |= .f' sample.yml ``` will output ```yaml b: &meow purr a: *meow ``` ## Explode alias and anchor Given a sample.yml file of: ```yaml f: a: &a cat b: *a ``` then ```bash yq 'explode(.f)' sample.yml ``` will output ```yaml f: a: cat b: cat ``` ## Explode with no aliases or anchors Given a sample.yml file of: ```yaml a: mike ``` then ```bash yq 'explode(.a)' sample.yml ``` will output ```yaml a: mike ``` ## Explode with alias keys Given a sample.yml file of: ```yaml f: a: &a cat *a : b ``` then ```bash yq 'explode(.f)' sample.yml ``` will output ```yaml f: a: cat cat: b ``` ## Dereference and update a field Use explode with multiply to dereference an object Given a sample.yml file of: ```yaml item_value: &item_value value: true thingOne: name: item_1 !!merge <<: *item_value thingTwo: name: item_2 !!merge <<: *item_value ``` then ```bash yq '.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}' sample.yml ``` will output ```yaml item_value: &item_value value: true thingOne: name: item_1 value: false thingTwo: name: item_2 !!merge <<: *item_value ``` ## LEGACY: Explode with merge anchors Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025 Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq 'explode(.)' sample.yml ``` will output ```yaml foo: a: foo_a thing: foo_thing c: foo_c bar: b: bar_b thing: bar_thing c: bar_c foobarList: b: bar_b thing: foo_thing c: foobarList_c a: foo_a foobar: c: foo_c a: foo_a thing: foobar_thing ``` ## LEGACY: Merge multiple maps see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order. Given a sample.yml file of: ```yaml - &CENTRE x: 1 y: 2 - &LEFT x: 0 y: 2 - &BIG r: 10 - &SMALL r: 1 - !!merge <<: - *CENTRE - *BIG ``` then ```bash yq '.[4] | explode(.)' sample.yml ``` will output ```yaml r: 10 x: 1 y: 2 ``` ## LEGACY: Override see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order. Given a sample.yml file of: ```yaml - &CENTRE x: 1 y: 2 - &LEFT x: 0 y: 2 - &BIG r: 10 - &SMALL r: 1 - !!merge <<: - *BIG - *LEFT - *SMALL x: 1 ``` then ```bash yq '.[4] | explode(.)' sample.yml ``` will output ```yaml r: 10 x: 1 y: 2 ``` ## FIXED: Explode with merge anchors Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). Observe that foobarList.b property is still foobarList_b. Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq 'explode(.)' sample.yml ``` will output ```yaml foo: a: foo_a thing: foo_thing c: foo_c bar: b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b a: foo_a thing: foo_thing c: foobarList_c foobar: c: foobar_c a: foo_a thing: foobar_thing ``` ## FIXED: Merge multiple maps Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order. Given a sample.yml file of: ```yaml - &CENTRE x: 1 y: 2 - &LEFT x: 0 y: 2 - &BIG r: 10 - &SMALL r: 1 - !!merge <<: - *CENTRE - *BIG ``` then ```bash yq '.[4] | explode(.)' sample.yml ``` will output ```yaml x: 1 y: 2 r: 10 ``` ## FIXED: Override Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order. Given a sample.yml file of: ```yaml - &CENTRE x: 1 y: 2 - &LEFT x: 0 y: 2 - &BIG r: 10 - &SMALL r: 1 - !!merge <<: - *BIG - *LEFT - *SMALL x: 1 ``` then ```bash yq '.[4] | explode(.)' sample.yml ``` will output ```yaml r: 10 y: 2 x: 1 ``` ## Exploding inline merge anchor Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025). Given a sample.yml file of: ```yaml a: b: &b 42 !!merge <<: c: *b ``` then ```bash yq 'explode(.) | sort_keys(.)' sample.yml ``` will output ```yaml a: b: 42 c: 42 ``` ================================================ FILE: pkg/yqlib/doc/operators/array-to-map.md ================================================ # Array to Map Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over. Behind the scenes, this is implemented using reduce: ``` (.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i) ``` ## Simple example Given a sample.yml file of: ```yaml cool: - null - null - hello ``` then ```bash yq '.cool |= array_to_map' sample.yml ``` will output ```yaml cool: 2: hello ``` ================================================ FILE: pkg/yqlib/doc/operators/assign-update.md ================================================ # Assign (Update) This operator is used to update node values. It can be used in either the: ### plain form: `=` Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline. ### relative form: `|=` This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment. ### Flags - `c` clobber custom tags ## Create yaml file Running ```bash yq --null-input '.a.b = "cat" | .x = "frog"' ``` will output ```yaml a: b: cat x: frog ``` ## Update node to be the child value Given a sample.yml file of: ```yaml a: b: g: foof ``` then ```bash yq '.a |= .b' sample.yml ``` will output ```yaml a: g: foof ``` ## Double elements in an array Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq '.[] |= . * 2' sample.yml ``` will output ```yaml - 2 - 4 - 6 ``` ## Update node from another file Note this will also work when the second file is a scalar (string/number) Given a sample.yml file of: ```yaml a: apples ``` And another sample another.yml file of: ```yaml b: bob ``` then ```bash yq eval-all 'select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)' sample.yml another.yml ``` will output ```yaml a: b: bob ``` ## Update node to be the sibling value Given a sample.yml file of: ```yaml a: b: child b: sibling ``` then ```bash yq '.a = .b' sample.yml ``` will output ```yaml a: sibling b: sibling ``` ## Updated multiple paths Given a sample.yml file of: ```yaml a: fieldA b: fieldB c: fieldC ``` then ```bash yq '(.a, .c) = "potato"' sample.yml ``` will output ```yaml a: potato b: fieldB c: potato ``` ## Update string value Given a sample.yml file of: ```yaml a: b: apple ``` then ```bash yq '.a.b = "frog"' sample.yml ``` will output ```yaml a: b: frog ``` ## Update string value via |= Note there is no difference between `=` and `|=` when the RHS is a scalar Given a sample.yml file of: ```yaml a: b: apple ``` then ```bash yq '.a.b |= "frog"' sample.yml ``` will output ```yaml a: b: frog ``` ## Update deeply selected results Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet. Given a sample.yml file of: ```yaml a: b: apple c: cactus ``` then ```bash yq '(.a[] | select(. == "apple")) = "frog"' sample.yml ``` will output ```yaml a: b: frog c: cactus ``` ## Update array values Given a sample.yml file of: ```yaml - candy - apple - sandy ``` then ```bash yq '(.[] | select(. == "*andy")) = "bogs"' sample.yml ``` will output ```yaml - bogs - apple - bogs ``` ## Update empty object Given a sample.yml file of: ```yaml {} ``` then ```bash yq '.a.b |= "bogs"' sample.yml ``` will output ```yaml a: b: bogs ``` ## Update node value that has an anchor Anchor will remain Given a sample.yml file of: ```yaml a: &cool cat ``` then ```bash yq '.a = "dog"' sample.yml ``` will output ```yaml a: &cool dog ``` ## Update empty object and array Given a sample.yml file of: ```yaml {} ``` then ```bash yq '.a.b.[0] |= "bogs"' sample.yml ``` will output ```yaml a: b: - bogs ``` ## Custom types are maintained by default Given a sample.yml file of: ```yaml a: !cat meow b: !dog woof ``` then ```bash yq '.a = .b' sample.yml ``` will output ```yaml a: !cat woof b: !dog woof ``` ## Custom types: clobber Use the `c` option to clobber custom tags Given a sample.yml file of: ```yaml a: !cat meow b: !dog woof ``` then ```bash yq '.a =c .b' sample.yml ``` will output ```yaml a: !dog woof b: !dog woof ``` ================================================ FILE: pkg/yqlib/doc/operators/boolean-operators.md ================================================ # Boolean Operators The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. `any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true. `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet These are most commonly used with the `select` operator to filter particular nodes. ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ## `or` example Running ```bash yq --null-input 'true or false' ``` will output ```yaml true ``` ## "yes" and "no" are strings In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/ Given a sample.yml file of: ```yaml - yes - no ``` then ```bash yq '.[] | tag' sample.yml ``` will output ```yaml !!str !!str ``` ## `and` example Running ```bash yq --null-input 'true and false' ``` will output ```yaml false ``` ## Matching nodes with select, equals and or Given a sample.yml file of: ```yaml - a: bird b: dog - a: frog b: bird - a: cat b: fly ``` then ```bash yq '[.[] | select(.a == "cat" or .b == "dog")]' sample.yml ``` will output ```yaml - a: bird b: dog - a: cat b: fly ``` ## `any` returns true if any boolean in a given array is true Given a sample.yml file of: ```yaml - false - true ``` then ```bash yq 'any' sample.yml ``` will output ```yaml true ``` ## `any` returns false for an empty array Given a sample.yml file of: ```yaml [] ``` then ```bash yq 'any' sample.yml ``` will output ```yaml false ``` ## `any_c` returns true if any element in the array is true for the given condition. Given a sample.yml file of: ```yaml a: - rad - awesome b: - meh - whatever ``` then ```bash yq '.[] |= any_c(. == "awesome")' sample.yml ``` will output ```yaml a: true b: false ``` ## `all` returns true if all booleans in a given array are true Given a sample.yml file of: ```yaml - true - true ``` then ```bash yq 'all' sample.yml ``` will output ```yaml true ``` ## `all` returns true for an empty array Given a sample.yml file of: ```yaml [] ``` then ```bash yq 'all' sample.yml ``` will output ```yaml true ``` ## `all_c` returns true if all elements in the array are true for the given condition. Given a sample.yml file of: ```yaml a: - rad - awesome b: - meh - 12 ``` then ```bash yq '.[] |= all_c(tag == "!!str")' sample.yml ``` will output ```yaml a: true b: false ``` ## Not true is false Running ```bash yq --null-input 'true | not' ``` will output ```yaml false ``` ## Not false is true Running ```bash yq --null-input 'false | not' ``` will output ```yaml true ``` ## String values considered to be true Running ```bash yq --null-input '"cat" | not' ``` will output ```yaml false ``` ## Empty string value considered to be true Running ```bash yq --null-input '"" | not' ``` will output ```yaml false ``` ## Numbers are considered to be true Running ```bash yq --null-input '1 | not' ``` will output ```yaml false ``` ## Zero is considered to be true Running ```bash yq --null-input '0 | not' ``` will output ```yaml false ``` ## Null is considered to be false Running ```bash yq --null-input '~ | not' ``` will output ```yaml true ``` ================================================ FILE: pkg/yqlib/doc/operators/collect-into-array.md ================================================ # Collect into Array This creates an array using the expression between the square brackets. ## Collect empty Running ```bash yq --null-input '[]' ``` will output ```yaml [] ``` ## Collect single Running ```bash yq --null-input '["cat"]' ``` will output ```yaml - cat ``` ## Collect many Given a sample.yml file of: ```yaml a: cat b: dog ``` then ```bash yq '[.a, .b]' sample.yml ``` will output ```yaml - cat - dog ``` ================================================ FILE: pkg/yqlib/doc/operators/column.md ================================================ # Column Returns the column of the matching node. Starts from 1, 0 indicates there was no column data. Column is the number of characters that precede that node on the line it starts. ## Returns column of _value_ node Given a sample.yml file of: ```yaml a: cat b: bob ``` then ```bash yq '.b | column' sample.yml ``` will output ```yaml 4 ``` ## Returns column of _key_ node Pipe through the key operator to get the column of the key Given a sample.yml file of: ```yaml a: cat b: bob ``` then ```bash yq '.b | key | column' sample.yml ``` will output ```yaml 1 ``` ## First column is 1 Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a | key | column' sample.yml ``` will output ```yaml 1 ``` ## No column data is 0 Running ```bash yq --null-input '{"a": "new entry"} | column' ``` will output ```yaml 0 ``` ================================================ FILE: pkg/yqlib/doc/operators/comment-operators.md ================================================ # Comment Operators Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples. Like the `=` and `|=` assign operators, the same syntax applies when updating comments: ### plain form: `=` This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline ### relative form: `|=` This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path. ## Set line comment Set the comment on the key node for more reliability (see below). Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a line_comment="single"' sample.yml ``` will output ```yaml a: cat # single ``` ## Set line comment of a maps/arrays For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars. Given a sample.yml file of: ```yaml a: b: things ``` then ```bash yq '(.a | key) line_comment="single"' sample.yml ``` will output ```yaml a: # single b: things ``` ## Use update assign to perform relative updates Given a sample.yml file of: ```yaml a: cat b: dog ``` then ```bash yq '.. line_comment |= .' sample.yml ``` will output ```yaml a: cat # cat b: dog # dog ``` ## Where is the comment - map key example The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). From this, you can see the 'hello-world-comment' is actually on the 'hello' key Given a sample.yml file of: ```yaml hello: # hello-world-comment message: world ``` then ```bash yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml ``` will output ```yaml - p: "" isKey: false hc: "" lc: "" fc: "" - p: hello isKey: true hc: "" lc: hello-world-comment fc: "" - p: hello isKey: false hc: "" lc: "" fc: "" - p: hello.message isKey: true hc: "" lc: "" fc: "" - p: hello.message isKey: false hc: "" lc: "" fc: "" ``` ## Retrieve comment - map key example From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment Given a sample.yml file of: ```yaml hello: # hello-world-comment message: world ``` then ```bash yq '.hello | key | line_comment' sample.yml ``` will output ```yaml hello-world-comment ``` ## Where is the comment - array example The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value). From this, you can see the 'under-name-comment' is actually on the first child Given a sample.yml file of: ```yaml name: # under-name-comment - first-array-child ``` then ```bash yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml ``` will output ```yaml - p: "" isKey: false hc: "" lc: "" fc: "" - p: name isKey: true hc: "" lc: "" fc: "" - p: name isKey: false hc: "" lc: "" fc: "" - p: name.0 isKey: false hc: under-name-comment lc: "" fc: "" ``` ## Retrieve comment - array example From the previous example, we know that the comment is on the first child as a headComment Given a sample.yml file of: ```yaml name: # under-name-comment - first-array-child ``` then ```bash yq '.name[0] | headComment' sample.yml ``` will output ```yaml under-name-comment ``` ## Set head comment Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '. head_comment="single"' sample.yml ``` will output ```yaml # single a: cat ``` ## Set head comment of a map entry Given a sample.yml file of: ```yaml f: foo a: b: cat ``` then ```bash yq '(.a | key) head_comment="single"' sample.yml ``` will output ```yaml f: foo # single a: b: cat ``` ## Set foot comment, using an expression Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '. foot_comment=.a' sample.yml ``` will output ```yaml a: cat # cat ``` ## Remove comment Given a sample.yml file of: ```yaml a: cat # comment b: dog # leave this ``` then ```bash yq '.a line_comment=""' sample.yml ``` will output ```yaml a: cat b: dog # leave this ``` ## Remove (strip) all comments Note the use of `...` to ensure key nodes are included. Given a sample.yml file of: ```yaml # hi a: cat # comment # great b: # key comment ``` then ```bash yq '... comments=""' sample.yml ``` will output ```yaml a: cat b: ``` ## Get line comment Given a sample.yml file of: ```yaml # welcome! a: cat # meow # have a great day ``` then ```bash yq '.a | line_comment' sample.yml ``` will output ```yaml meow ``` ## Get head comment Given a sample.yml file of: ```yaml # welcome! a: cat # meow # have a great day ``` then ```bash yq '. | head_comment' sample.yml ``` will output ```yaml welcome! ``` ## Head comment with document split Given a sample.yml file of: ```yaml # welcome! --- # bob a: cat # meow # have a great day ``` then ```bash yq 'head_comment' sample.yml ``` will output ```yaml welcome! bob ``` ## Get foot comment Given a sample.yml file of: ```yaml # welcome! a: cat # meow # have a great day # no really ``` then ```bash yq '. | foot_comment' sample.yml ``` will output ```yaml have a great day no really ``` ================================================ FILE: pkg/yqlib/doc/operators/compare.md ================================================ # Compare Operators Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. The following types are currently supported: - numbers - strings - datetimes ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ## Compare numbers (>) Given a sample.yml file of: ```yaml a: 5 b: 4 ``` then ```bash yq '.a > .b' sample.yml ``` will output ```yaml true ``` ## Compare equal numbers (>=) Given a sample.yml file of: ```yaml a: 5 b: 5 ``` then ```bash yq '.a >= .b' sample.yml ``` will output ```yaml true ``` ## Compare strings Compares strings by their bytecode. Given a sample.yml file of: ```yaml a: zoo b: apple ``` then ```bash yq '.a > .b' sample.yml ``` will output ```yaml true ``` ## Compare date times You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. Given a sample.yml file of: ```yaml a: 2021-01-01T03:10:00Z b: 2020-01-01T03:10:00Z ``` then ```bash yq '.a > .b' sample.yml ``` will output ```yaml true ``` ## Both sides are null: > is false Running ```bash yq --null-input '.a > .b' ``` will output ```yaml false ``` ## Both sides are null: >= is true Running ```bash yq --null-input '.a >= .b' ``` will output ```yaml true ``` ================================================ FILE: pkg/yqlib/doc/operators/contains.md ================================================ # Contains This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring. {% hint style="warning" %} _Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains(["cat"])` will return true for an array `["cats"]`. See the "Array has a subset array" example below on how to check for a subset. {% endhint %} ## Array contains array Array is equal or subset of Given a sample.yml file of: ```yaml - foobar - foobaz - blarp ``` then ```bash yq 'contains(["baz", "bar"])' sample.yml ``` will output ```yaml true ``` ## Array has a subset array Subtract the superset array from the subset, if there's anything left, it's not a subset Given a sample.yml file of: ```yaml - foobar - foobaz - blarp ``` then ```bash yq '["baz", "bar"] - . | length == 0' sample.yml ``` will output ```yaml false ``` ## Object included in array Given a sample.yml file of: ```yaml "foo": 12 "bar": - 1 - 2 - "barp": 12 "blip": 13 ``` then ```bash yq 'contains({"bar": [{"barp": 12}]})' sample.yml ``` will output ```yaml true ``` ## Object not included in array Given a sample.yml file of: ```yaml "foo": 12 "bar": - 1 - 2 - "barp": 12 "blip": 13 ``` then ```bash yq 'contains({"foo": 12, "bar": [{"barp": 15}]})' sample.yml ``` will output ```yaml false ``` ## String contains substring Given a sample.yml file of: ```yaml foobar ``` then ```bash yq 'contains("bar")' sample.yml ``` will output ```yaml true ``` ## String equals string Given a sample.yml file of: ```yaml meow ``` then ```bash yq 'contains("meow")' sample.yml ``` will output ```yaml true ``` ================================================ FILE: pkg/yqlib/doc/operators/create-collect-into-object.md ================================================ # Create, Collect into Object This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents. ## Collect empty object Running ```bash yq --null-input '{}' ``` will output ```yaml {} ``` ## Wrap (prefix) existing object Given a sample.yml file of: ```yaml name: Mike ``` then ```bash yq '{"wrap": .}' sample.yml ``` will output ```yaml wrap: name: Mike ``` ## Using splat to create multiple objects Given a sample.yml file of: ```yaml name: Mike pets: - cat - dog ``` then ```bash yq '{.name: .pets.[]}' sample.yml ``` will output ```yaml Mike: cat Mike: dog ``` ## Working with multiple documents Given a sample.yml file of: ```yaml name: Mike pets: - cat - dog --- name: Rosey pets: - monkey - sheep ``` then ```bash yq '{.name: .pets.[]}' sample.yml ``` will output ```yaml Mike: cat Mike: dog --- Rosey: monkey Rosey: sheep ``` ## Creating yaml from scratch Running ```bash yq --null-input '{"wrap": "frog"}' ``` will output ```yaml wrap: frog ``` ## Creating yaml from scratch with multiple objects Running ```bash yq --null-input '(.a.b = "foo") | (.d.e = "bar")' ``` will output ```yaml a: b: foo d: e: bar ``` ================================================ FILE: pkg/yqlib/doc/operators/datetime.md ================================================ # Date Time Various operators for parsing and manipulating dates. ## Date time formatting This uses Golang's built in time library for parsing and formatting date times. When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. ```bash yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' ``` See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options. ## Timezones This uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details. ## Durations Durations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. You can add durations to time using the `+` operator. ## Format: from standard RFC3339 format Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node. Given a sample.yml file of: ```yaml a: 2001-12-15T02:59:43.1Z ``` then ```bash yq '.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 2:59AM ``` ## Format: from custom date time Use with_dtf to set a custom datetime format for parsing. Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM ``` then ```bash yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))' sample.yml ``` will output ```yaml a: 2001-12-15 ``` ## Format: get the day of the week Given a sample.yml file of: ```yaml a: 2001-12-15 ``` then ```bash yq '.a | format_datetime("Monday")' sample.yml ``` will output ```yaml Saturday ``` ## Now Given a sample.yml file of: ```yaml a: cool ``` then ```bash yq '.updated = now' sample.yml ``` will output ```yaml a: cool updated: 2021-05-19T01:02:03Z ``` ## From Unix Converts from unix time. Note, you don't have to pipe through the tz operator :) Running ```bash yq --null-input '1675301929 | from_unix | tz("UTC")' ``` will output ```yaml 2023-02-02T01:38:49Z ``` ## To Unix Converts to unix time Running ```bash yq --null-input 'now | to_unix' ``` will output ```yaml 1621386123 ``` ## Timezone: from standard RFC3339 format Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format. Given a sample.yml file of: ```yaml a: cool ``` then ```bash yq '.updated = (now | tz("Australia/Sydney"))' sample.yml ``` will output ```yaml a: cool updated: 2021-05-19T11:02:03+10:00 ``` ## Timezone: with custom format Specify standard IANA Time Zone format or 'utc', 'local' Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 1:59PM AEDT ``` ## Add and tz custom format Specify standard IANA Time Zone format or 'utc', 'local' Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash yq '.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 1:59PM AEDT ``` ## Date addition Given a sample.yml file of: ```yaml a: 2021-01-01T00:00:00Z ``` then ```bash yq '.a += "3h10m"' sample.yml ``` will output ```yaml a: 2021-01-01T03:10:00Z ``` ## Date subtraction You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information. Given a sample.yml file of: ```yaml a: 2021-01-01T03:10:00Z ``` then ```bash yq '.a -= "3h10m"' sample.yml ``` will output ```yaml a: 2021-01-01T00:00:00Z ``` ## Date addition - custom format Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 6:00AM GMT ``` ## Date script with custom format You can embed full expressions in with_dtf if needed. Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` then ```bash yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 2:00PM AWST ``` ================================================ FILE: pkg/yqlib/doc/operators/delete.md ================================================ # Delete Deletes matching entries in maps or arrays. ## Delete entry in map Given a sample.yml file of: ```yaml a: cat b: dog ``` then ```bash yq 'del(.b)' sample.yml ``` will output ```yaml a: cat ``` ## Delete nested entry in map Given a sample.yml file of: ```yaml a: a1: fred a2: frood ``` then ```bash yq 'del(.a.a1)' sample.yml ``` will output ```yaml a: a2: frood ``` ## Delete entry in array Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq 'del(.[1])' sample.yml ``` will output ```yaml - 1 - 3 ``` ## Delete nested entry in array Given a sample.yml file of: ```yaml - a: cat b: dog ``` then ```bash yq 'del(.[0].a)' sample.yml ``` will output ```yaml - b: dog ``` ## Delete no matches Given a sample.yml file of: ```yaml a: cat b: dog ``` then ```bash yq 'del(.c)' sample.yml ``` will output ```yaml a: cat b: dog ``` ## Delete matching entries Given a sample.yml file of: ```yaml a: cat b: dog c: bat ``` then ```bash yq 'del( .[] | select(. == "*at") )' sample.yml ``` will output ```yaml b: dog ``` ## Recursively delete matching keys Given a sample.yml file of: ```yaml a: name: frog b: name: blog age: 12 ``` then ```bash yq 'del(.. | select(has("name")).name)' sample.yml ``` will output ```yaml a: b: age: 12 ``` ================================================ FILE: pkg/yqlib/doc/operators/divide.md ================================================ # Divide Divide behaves differently according to the type of the LHS: * strings: split by the divider * number: arithmetic division ## String split Given a sample.yml file of: ```yaml a: cat_meow b: _ ``` then ```bash yq '.c = .a / .b' sample.yml ``` will output ```yaml a: cat_meow b: _ c: - cat - meow ``` ## Number division The result during division is calculated as a float Given a sample.yml file of: ```yaml a: 12 b: 2.5 ``` then ```bash yq '.a = .a / .b' sample.yml ``` will output ```yaml a: 4.8 b: 2.5 ``` ## Number division by zero Dividing by zero results in +Inf or -Inf Given a sample.yml file of: ```yaml a: 1 b: -1 ``` then ```bash yq '.a = .a / 0 | .b = .b / 0' sample.yml ``` will output ```yaml a: !!float +Inf b: !!float -Inf ``` ================================================ FILE: pkg/yqlib/doc/operators/document-index.md ================================================ # Document Index Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document. ## Retrieve a document index Given a sample.yml file of: ```yaml a: cat --- a: frog ``` then ```bash yq '.a | document_index' sample.yml ``` will output ```yaml 0 --- 1 ``` ## Retrieve a document index, shorthand Given a sample.yml file of: ```yaml a: cat --- a: frog ``` then ```bash yq '.a | di' sample.yml ``` will output ```yaml 0 --- 1 ``` ## Filter by document index Given a sample.yml file of: ```yaml a: cat --- a: frog ``` then ```bash yq 'select(document_index == 1)' sample.yml ``` will output ```yaml a: frog ``` ## Filter by document index shorthand Given a sample.yml file of: ```yaml a: cat --- a: frog ``` then ```bash yq 'select(di == 1)' sample.yml ``` will output ```yaml a: frog ``` ## Print Document Index with matches Given a sample.yml file of: ```yaml a: cat --- a: frog ``` then ```bash yq '.a | ({"match": ., "doc": document_index})' sample.yml ``` will output ```yaml match: cat doc: 0 --- match: frog doc: 1 ``` ================================================ FILE: pkg/yqlib/doc/operators/encode-decode.md ================================================ # Encoder / Decoder Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. Note that you can optionally pass an indent value to the encode functions (see below). These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them. | Format | Decode (from string) | Encode (to string) | | --- | -- | --| | Yaml | from_yaml/@yamld | to_yaml(i)/@yaml | | JSON | from_json/@jsond | to_json(i)/@json | | Properties | from_props/@propsd | to_props/@props | | CSV | from_csv/@csvd | to_csv/@csv | | TSV | from_tsv/@tsvd | to_tsv/@tsv | | XML | from_xml/@xmld | to_xml(i)/@xml | | Base64 | @base64d | @base64 | | URI | @urid | @uri | | Shell | | @sh | See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content. ## Encode value as json string Given a sample.yml file of: ```yaml a: cool: thing ``` then ```bash yq '.b = (.a | to_json)' sample.yml ``` will output ```yaml a: cool: thing b: | { "cool": "thing" } ``` ## Encode value as json string, on one line Pass in a 0 indent to print json on a single line. Given a sample.yml file of: ```yaml a: cool: thing ``` then ```bash yq '.b = (.a | to_json(0))' sample.yml ``` will output ```yaml a: cool: thing b: '{"cool":"thing"}' ``` ## Encode value as json string, on one line shorthand Pass in a 0 indent to print json on a single line. Given a sample.yml file of: ```yaml a: cool: thing ``` then ```bash yq '.b = (.a | @json)' sample.yml ``` will output ```yaml a: cool: thing b: '{"cool":"thing"}' ``` ## Decode a json encoded string Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling. Given a sample.yml file of: ```yaml a: '{"cool":"thing"}' ``` then ```bash yq '.a | from_json | ... style=""' sample.yml ``` will output ```yaml cool: thing ``` ## Encode value as props string Given a sample.yml file of: ```yaml a: cool: thing ``` then ```bash yq '.b = (.a | @props)' sample.yml ``` will output ```yaml a: cool: thing b: | cool = thing ``` ## Decode props encoded string Given a sample.yml file of: ```yaml a: |- cats=great dogs=cool as well ``` then ```bash yq '.a |= @propsd' sample.yml ``` will output ```yaml a: cats: great dogs: cool as well ``` ## Decode csv encoded string Given a sample.yml file of: ```yaml a: |- cats,dogs great,cool as well ``` then ```bash yq '.a |= @csvd' sample.yml ``` will output ```yaml a: - cats: great dogs: cool as well ``` ## Decode tsv encoded string Given a sample.yml file of: ```yaml a: |- cats dogs great cool as well ``` then ```bash yq '.a |= @tsvd' sample.yml ``` will output ```yaml a: - cats: great dogs: cool as well ``` ## Encode value as yaml string Indent defaults to 2 Given a sample.yml file of: ```yaml a: cool: bob: dylan ``` then ```bash yq '.b = (.a | to_yaml)' sample.yml ``` will output ```yaml a: cool: bob: dylan b: | cool: bob: dylan ``` ## Encode value as yaml string, with custom indentation You can specify the indentation level as the first parameter. Given a sample.yml file of: ```yaml a: cool: bob: dylan ``` then ```bash yq '.b = (.a | to_yaml(8))' sample.yml ``` will output ```yaml a: cool: bob: dylan b: | cool: bob: dylan ``` ## Decode a yaml encoded string Given a sample.yml file of: ```yaml a: 'foo: bar' ``` then ```bash yq '.b = (.a | from_yaml)' sample.yml ``` will output ```yaml a: 'foo: bar' b: foo: bar ``` ## Update a multiline encoded yaml string Given a sample.yml file of: ```yaml a: | foo: bar baz: dog ``` then ```bash yq '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml ``` will output ```yaml a: | foo: cat baz: dog ``` ## Update a single line encoded yaml string Given a sample.yml file of: ```yaml a: 'foo: bar' ``` then ```bash yq '.a |= (from_yaml | .foo = "cat" | to_yaml)' sample.yml ``` will output ```yaml a: 'foo: cat' ``` ## Encode array of scalars as csv string Scalars are strings, numbers and booleans. Given a sample.yml file of: ```yaml - cat - thing1,thing2 - true - 3.40 ``` then ```bash yq '@csv' sample.yml ``` will output ```yaml cat,"thing1,thing2",true,3.40 ``` ## Encode array of arrays as csv string Given a sample.yml file of: ```yaml - - cat - thing1,thing2 - true - 3.40 - - dog - thing3 - false - 12 ``` then ```bash yq '@csv' sample.yml ``` will output ```yaml cat,"thing1,thing2",true,3.40 dog,thing3,false,12 ``` ## Encode array of arrays as tsv string Scalars are strings, numbers and booleans. Given a sample.yml file of: ```yaml - - cat - thing1,thing2 - true - 3.40 - - dog - thing3 - false - 12 ``` then ```bash yq '@tsv' sample.yml ``` will output ```yaml cat thing1,thing2 true 3.40 dog thing3 false 12 ``` ## Encode value as xml string Given a sample.yml file of: ```yaml a: cool: foo: bar +@id: hi ``` then ```bash yq '.a | to_xml' sample.yml ``` will output ```yaml bar ``` ## Encode value as xml string on a single line Given a sample.yml file of: ```yaml a: cool: foo: bar +@id: hi ``` then ```bash yq '.a | @xml' sample.yml ``` will output ```yaml bar ``` ## Encode value as xml string with custom indentation Given a sample.yml file of: ```yaml a: cool: foo: bar +@id: hi ``` then ```bash yq '{"cat": .a | to_xml(1)}' sample.yml ``` will output ```yaml cat: | bar ``` ## Decode a xml encoded string Given a sample.yml file of: ```yaml a: bar ``` then ```bash yq '.b = (.a | from_xml)' sample.yml ``` will output ```yaml a: bar b: foo: bar ``` ## Encode a string to base64 Given a sample.yml file of: ```yaml coolData: a special string ``` then ```bash yq '.coolData | @base64' sample.yml ``` will output ```yaml YSBzcGVjaWFsIHN0cmluZw== ``` ## Encode a yaml document to base64 Pipe through @yaml first to convert to a string, then use @base64 to encode it. Given a sample.yml file of: ```yaml a: apple ``` then ```bash yq '@yaml | @base64' sample.yml ``` will output ```yaml YTogYXBwbGUK ``` ## Encode a string to uri Given a sample.yml file of: ```yaml coolData: this has & special () characters * ``` then ```bash yq '.coolData | @uri' sample.yml ``` will output ```yaml this+has+%26+special+%28%29+characters+%2A ``` ## Decode a URI to a string Given a sample.yml file of: ```yaml this+has+%26+special+%28%29+characters+%2A ``` then ```bash yq '@urid' sample.yml ``` will output ```yaml this has & special () characters * ``` ## Encode a string to sh Sh/Bash friendly string Given a sample.yml file of: ```yaml coolData: strings with spaces and a 'quote' ``` then ```bash yq '.coolData | @sh' sample.yml ``` will output ```yaml strings' with spaces and a '\'quote\' ``` ## Decode a base64 encoded string Decoded data is assumed to be a string. Given a sample.yml file of: ```yaml coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig== ``` then ```bash yq '.coolData | @base64d' sample.yml ``` will output ```yaml Works with UTF-16 😊 ``` ## Decode a base64 encoded yaml document Pipe through `from_yaml` to parse the decoded base64 string as a yaml document. Given a sample.yml file of: ```yaml coolData: YTogYXBwbGUK ``` then ```bash yq '.coolData |= (@base64d | from_yaml)' sample.yml ``` will output ```yaml coolData: a: apple ``` ================================================ FILE: pkg/yqlib/doc/operators/entries.md ================================================ # Entries Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. Use `with_entries(op)` as a syntactic sugar for doing `to_entries | op | from_entries`. ## to_entries Map Given a sample.yml file of: ```yaml a: 1 b: 2 ``` then ```bash yq 'to_entries' sample.yml ``` will output ```yaml - key: a value: 1 - key: b value: 2 ``` ## to_entries Array Given a sample.yml file of: ```yaml - a - b ``` then ```bash yq 'to_entries' sample.yml ``` will output ```yaml - key: 0 value: a - key: 1 value: b ``` ## to_entries null Given a sample.yml file of: ```yaml null ``` then ```bash yq 'to_entries' sample.yml ``` will output ```yaml ``` ## from_entries map Given a sample.yml file of: ```yaml a: 1 b: 2 ``` then ```bash yq 'to_entries | from_entries' sample.yml ``` will output ```yaml a: 1 b: 2 ``` ## from_entries with numeric key indices from_entries always creates a map, even for numeric keys Given a sample.yml file of: ```yaml - a - b ``` then ```bash yq 'to_entries | from_entries' sample.yml ``` will output ```yaml 0: a 1: b ``` ## Use with_entries to update keys Given a sample.yml file of: ```yaml a: 1 b: 2 ``` then ```bash yq 'with_entries(.key |= "KEY_" + .)' sample.yml ``` will output ```yaml KEY_a: 1 KEY_b: 2 ``` ## Use with_entries to update keys recursively We use (.. | select(tag="map")) to find all the maps in the doc, then |= to update each one of those maps. In the update, with_entries is used. Given a sample.yml file of: ```yaml a: 1 b: b_a: nested b_b: thing ``` then ```bash yq '(.. | select(tag=="!!map")) |= with_entries(.key |= "KEY_" + .)' sample.yml ``` will output ```yaml KEY_a: 1 KEY_b: KEY_b_a: nested KEY_b_b: thing ``` ## Custom sort map keys Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back. Given a sample.yml file of: ```yaml a: 1 c: 3 b: 2 ``` then ```bash yq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml ``` will output ```yaml c: 3 b: 2 a: 1 ``` ## Use with_entries to filter the map Given a sample.yml file of: ```yaml a: b: bird c: d: dog ``` then ```bash yq 'with_entries(select(.value | has("b")))' sample.yml ``` will output ```yaml a: b: bird ``` ================================================ FILE: pkg/yqlib/doc/operators/env-variable-operators.md ================================================ # Env Variable Operators These operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. There are three operators: - `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) - `strenv` which also takes a single environment variable name, and always parses the variable as a string. - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). ## EnvSubst Options You can optionally pass envsubst any of the following options: - nu: NoUnset, this will fail if there are any referenced variables that are not set - ne: NoEmpty, this will fail if there are any referenced variables that are empty - ff: FailFast, this will abort on the first failure (rather than collect all the errors) E.g: `envsubst(ne, ff)` will fail on the first empty variable. See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. ## Tip To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator as follows: ```bash yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml ``` ## Disabling env operators If required, you can use the `--security-disable-env-ops` to disable env operations. ## Read string environment variable Running ```bash myenv="cat meow" yq --null-input '.a = env(myenv)' ``` will output ```yaml a: cat meow ``` ## Read boolean environment variable Running ```bash myenv="true" yq --null-input '.a = env(myenv)' ``` will output ```yaml a: true ``` ## Read numeric environment variable Running ```bash myenv="12" yq --null-input '.a = env(myenv)' ``` will output ```yaml a: 12 ``` ## Read yaml environment variable Running ```bash myenv="{b: fish}" yq --null-input '.a = env(myenv)' ``` will output ```yaml a: {b: fish} ``` ## Read boolean environment variable as a string Running ```bash myenv="true" yq --null-input '.a = strenv(myenv)' ``` will output ```yaml a: "true" ``` ## Read numeric environment variable as a string Running ```bash myenv="12" yq --null-input '.a = strenv(myenv)' ``` will output ```yaml a: "12" ``` ## Dynamically update a path from an environment variable The env variable can be any valid yq expression. Given a sample.yml file of: ```yaml a: b: - name: dog - name: cat ``` then ```bash pathEnv=".a.b[0].name" valueEnv="moo" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml ``` will output ```yaml a: b: - name: moo - name: cat ``` ## Dynamic key lookup with environment variable Given a sample.yml file of: ```yaml cat: meow dog: woof ``` then ```bash myenv="cat" yq '.[env(myenv)]' sample.yml ``` will output ```yaml meow ``` ## Replace strings with envsubst Running ```bash myenv="cat" yq --null-input '"the ${myenv} meows" | envsubst' ``` will output ```yaml the cat meows ``` ## Replace strings with envsubst, missing variables Running ```bash yq --null-input '"the ${myenvnonexisting} meows" | envsubst' ``` will output ```yaml the meows ``` ## Replace strings with envsubst(nu), missing variables (nu) not unset, will fail if there are unset (missing) variables Running ```bash yq --null-input '"the ${myenvnonexisting} meows" | envsubst(nu)' ``` will output ```bash Error: variable ${myenvnonexisting} not set ``` ## Replace strings with envsubst(ne), missing variables (ne) not empty, only validates set variables Running ```bash yq --null-input '"the ${myenvnonexisting} meows" | envsubst(ne)' ``` will output ```yaml the meows ``` ## Replace strings with envsubst(ne), empty variable (ne) not empty, will fail if a references variable is empty Running ```bash myenv="" yq --null-input '"the ${myenv} meows" | envsubst(ne)' ``` will output ```bash Error: variable ${myenv} set but empty ``` ## Replace strings with envsubst, missing variables with defaults Running ```bash yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst' ``` will output ```yaml the dog meows ``` ## Replace strings with envsubst(nu), missing variables with defaults Having a default specified skips over the missing variable. Running ```bash yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst(nu)' ``` will output ```yaml the dog meows ``` ## Replace strings with envsubst(ne), missing variables with defaults Fails, because the variable is explicitly set to blank. Running ```bash myEmptyEnv="" yq --null-input '"the ${myEmptyEnv-dog} meows" | envsubst(ne)' ``` will output ```bash Error: variable ${myEmptyEnv} set but empty ``` ## Replace string environment variable in document Given a sample.yml file of: ```yaml v: ${myenv} ``` then ```bash myenv="cat meow" yq '.v |= envsubst' sample.yml ``` will output ```yaml v: cat meow ``` ## (Default) Return all envsubst errors By default, all errors are returned at once. Running ```bash yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu)' ``` will output ```bash Error: variable ${notThere} not set variable ${alsoNotThere} not set ``` ## Fail fast, return the first envsubst error (and abort) Running ```bash yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)' ``` will output ```bash Error: variable ${notThere} not set ``` ## env() operation fails when security is enabled Use `--security-disable-env-ops` to disable env operations for security. Running ```bash yq --null-input 'env("MYENV")' ``` will output ```bash Error: env operations have been disabled ``` ## strenv() operation fails when security is enabled Use `--security-disable-env-ops` to disable env operations for security. Running ```bash yq --null-input 'strenv("MYENV")' ``` will output ```bash Error: env operations have been disabled ``` ## envsubst() operation fails when security is enabled Use `--security-disable-env-ops` to disable env operations for security. Running ```bash yq --null-input '"value: ${MYENV}" | envsubst' ``` will output ```bash Error: env operations have been disabled ``` ================================================ FILE: pkg/yqlib/doc/operators/equals.md ================================================ # Equals / Not Equals This is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise. ``` .a == .b ``` It is most often used with the select operator to find particular nodes: ``` select(.a == .b) ``` The not equals `!=` operator returns `false` if the LHS is equal to the RHS. ## Related Operators - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ## Match string Given a sample.yml file of: ```yaml - cat - goat - dog ``` then ```bash yq '.[] | (. == "*at")' sample.yml ``` will output ```yaml true true false ``` ## Don't match string Given a sample.yml file of: ```yaml - cat - goat - dog ``` then ```bash yq '.[] | (. != "*at")' sample.yml ``` will output ```yaml false false true ``` ## Match number Given a sample.yml file of: ```yaml - 3 - 4 - 5 ``` then ```bash yq '.[] | (. == 4)' sample.yml ``` will output ```yaml false true false ``` ## Don't match number Given a sample.yml file of: ```yaml - 3 - 4 - 5 ``` then ```bash yq '.[] | (. != 4)' sample.yml ``` will output ```yaml true false true ``` ## Match nulls Running ```bash yq --null-input 'null == ~' ``` will output ```yaml true ``` ## Non existent key doesn't equal a value Given a sample.yml file of: ```yaml a: frog ``` then ```bash yq 'select(.b != "thing")' sample.yml ``` will output ```yaml a: frog ``` ## Two non existent keys are equal Given a sample.yml file of: ```yaml a: frog ``` then ```bash yq 'select(.b == .c)' sample.yml ``` will output ```yaml a: frog ``` ================================================ FILE: pkg/yqlib/doc/operators/error.md ================================================ # Error Use this operation to short-circuit expressions. Useful for validation. ## Validate a particular value Given a sample.yml file of: ```yaml a: hello ``` then ```bash yq 'select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")' sample.yml ``` will output ```bash Error: .a [hello] is not howdy! ``` ## Validate the environment variable is a number - invalid Running ```bash numberOfCats="please" yq --null-input 'env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")' ``` will output ```bash Error: numberOfCats is not a number :( ``` ## Validate the environment variable is a number - valid `with` can be a convenient way of encapsulating validation. Given a sample.yml file of: ```yaml name: Bob favouriteAnimal: cat ``` then ```bash numberOfCats="3" yq ' with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | .numPets = env(numberOfCats) ' sample.yml ``` will output ```yaml name: Bob favouriteAnimal: cat numPets: 3 ``` ================================================ FILE: pkg/yqlib/doc/operators/eval.md ================================================ # Eval Use `eval` to dynamically process an expression - for instance from an environment variable. `eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. Tip: This can be a useful way to parameterise complex scripts. ## Dynamically evaluate a path Given a sample.yml file of: ```yaml pathExp: .a.b[] | select(.name == "cat") a: b: - name: dog - name: cat ``` then ```bash yq 'eval(.pathExp)' sample.yml ``` will output ```yaml name: cat ``` ## Dynamically update a path from an environment variable The env variable can be any valid yq expression. Given a sample.yml file of: ```yaml a: b: - name: dog - name: cat ``` then ```bash pathEnv=".a.b[0].name" valueEnv="moo" yq 'eval(strenv(pathEnv)) = strenv(valueEnv)' sample.yml ``` will output ```yaml a: b: - name: moo - name: cat ``` ================================================ FILE: pkg/yqlib/doc/operators/file-operators.md ================================================ # File Operators File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document). Note that the `fileIndex` operator has a short alias of `fi`. ## Merging files Note the use of eval-all to ensure all documents are loaded into memory. ```bash yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml ``` ## Get filename Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq 'filename' sample.yml ``` will output ```yaml sample.yml ``` ## Get file index Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq 'file_index' sample.yml ``` will output ```yaml 0 ``` ## Get file indices of multiple documents Given a sample.yml file of: ```yaml a: cat ``` And another sample another.yml file of: ```yaml a: cat ``` then ```bash yq eval-all 'file_index' sample.yml another.yml ``` will output ```yaml 0 1 ``` ## Get file index alias Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq 'fi' sample.yml ``` will output ```yaml 0 ``` ================================================ FILE: pkg/yqlib/doc/operators/filter.md ================================================ # Filter Filters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`. ## Filter array Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq 'filter(. < 3)' sample.yml ``` will output ```yaml - 1 - 2 ``` ## Filter map values Given a sample.yml file of: ```yaml c: things: cool frog: yes d: things: hot frog: false ``` then ```bash yq 'filter(.things == "cool")' sample.yml ``` will output ```yaml - things: cool frog: yes ``` ================================================ FILE: pkg/yqlib/doc/operators/first.md ================================================ # First Returns the first matching element in an array, or first matching value in a map. Can be given an expression to match with, otherwise will just return the first. ## First matching element from array Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'first(.a == "cat")' sample.yml ``` will output ```yaml a: cat ``` ## First matching element from array with multiple matches Given a sample.yml file of: ```yaml - a: banana - a: cat b: firstCat - a: apple - a: cat b: secondCat ``` then ```bash yq 'first(.a == "cat")' sample.yml ``` will output ```yaml a: cat b: firstCat ``` ## First matching element from array with numeric condition Given a sample.yml file of: ```yaml - a: 10 - a: 100 - a: 1 - a: 101 ``` then ```bash yq 'first(.a > 50)' sample.yml ``` will output ```yaml a: 100 ``` ## First matching element from array with boolean condition Given a sample.yml file of: ```yaml - a: false - a: true b: firstTrue - a: false - a: true b: secondTrue ``` then ```bash yq 'first(.a == true)' sample.yml ``` will output ```yaml a: true b: firstTrue ``` ## First matching element from array with null values Given a sample.yml file of: ```yaml - a: null - a: cat - a: apple ``` then ```bash yq 'first(.a != null)' sample.yml ``` will output ```yaml a: cat ``` ## First matching element from array with complex condition Given a sample.yml file of: ```yaml - a: dog b: 7 - a: cat b: 3 - a: apple b: 5 ``` then ```bash yq 'first(.b > 4 and .b < 6)' sample.yml ``` will output ```yaml a: apple b: 5 ``` ## First matching element from map Given a sample.yml file of: ```yaml x: a: banana y: a: cat z: a: apple ``` then ```bash yq 'first(.a == "cat")' sample.yml ``` will output ```yaml a: cat ``` ## First matching element from map with numeric condition Given a sample.yml file of: ```yaml x: a: 10 y: a: 100 z: a: 101 ``` then ```bash yq 'first(.a > 50)' sample.yml ``` will output ```yaml a: 100 ``` ## First matching element from nested structure Given a sample.yml file of: ```yaml items: - a: banana - a: cat - a: apple ``` then ```bash yq '.items | first(.a == "cat")' sample.yml ``` will output ```yaml a: cat ``` ## First matching element with no matches Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'first(.a == "dog")' sample.yml ``` will output ```yaml ``` ## First matching element from empty array Given a sample.yml file of: ```yaml [] ``` then ```bash yq 'first(.a == "cat")' sample.yml ``` will output ```yaml ``` ## First matching element from scalar node Given a sample.yml file of: ```yaml hello ``` then ```bash yq 'first(. == "hello")' sample.yml ``` will output ```yaml ``` ## First matching element from null node Given a sample.yml file of: ```yaml null ``` then ```bash yq 'first(. == "hello")' sample.yml ``` will output ```yaml ``` ## First matching element with string condition Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'first(.a | test("^c"))' sample.yml ``` will output ```yaml a: cat ``` ## First matching element with length condition Given a sample.yml file of: ```yaml - a: hi - a: hello - a: world ``` then ```bash yq 'first(.a | length > 4)' sample.yml ``` will output ```yaml a: hello ``` ## First matching element from array of strings Given a sample.yml file of: ```yaml - banana - cat - apple ``` then ```bash yq 'first(. == "cat")' sample.yml ``` will output ```yaml cat ``` ## First matching element from array of numbers Given a sample.yml file of: ```yaml - 10 - 100 - 1 ``` then ```bash yq 'first(. > 50)' sample.yml ``` will output ```yaml 100 ``` ## First element with no filter from array Given a sample.yml file of: ```yaml - 10 - 100 - 1 ``` then ```bash yq 'first' sample.yml ``` will output ```yaml 10 ``` ## First element with no filter from array of maps Given a sample.yml file of: ```yaml - a: 10 - a: 100 ``` then ```bash yq 'first' sample.yml ``` will output ```yaml a: 10 ``` ================================================ FILE: pkg/yqlib/doc/operators/flatten.md ================================================ # Flatten This recursively flattens arrays. ## Flatten Recursively flattens all arrays Given a sample.yml file of: ```yaml - 1 - - 2 - - - 3 ``` then ```bash yq 'flatten' sample.yml ``` will output ```yaml - 1 - 2 - 3 ``` ## Flatten with depth of one Given a sample.yml file of: ```yaml - 1 - - 2 - - - 3 ``` then ```bash yq 'flatten(1)' sample.yml ``` will output ```yaml - 1 - 2 - - 3 ``` ## Flatten empty array Given a sample.yml file of: ```yaml - [] ``` then ```bash yq 'flatten' sample.yml ``` will output ```yaml [] ``` ## Flatten array of objects Given a sample.yml file of: ```yaml - foo: bar - - foo: baz ``` then ```bash yq 'flatten' sample.yml ``` will output ```yaml - foo: bar - foo: baz ``` ================================================ FILE: pkg/yqlib/doc/operators/group-by.md ================================================ # Group By This is used to group items in an array by an expression. ## Group by field Given a sample.yml file of: ```yaml - foo: 1 bar: 10 - foo: 3 bar: 100 - foo: 1 bar: 1 ``` then ```bash yq 'group_by(.foo)' sample.yml ``` will output ```yaml - - foo: 1 bar: 10 - foo: 1 bar: 1 - - foo: 3 bar: 100 ``` ## Group by field, with nulls Given a sample.yml file of: ```yaml - cat: dog - foo: 1 bar: 10 - foo: 3 bar: 100 - no: foo for you - foo: 1 bar: 1 ``` then ```bash yq 'group_by(.foo)' sample.yml ``` will output ```yaml - - cat: dog - no: foo for you - - foo: 1 bar: 10 - foo: 1 bar: 1 - - foo: 3 bar: 100 ``` ================================================ FILE: pkg/yqlib/doc/operators/has.md ================================================ # Has This operation returns true if the key exists in a map (or index in an array), false otherwise. ## Has map key Given a sample.yml file of: ```yaml - a: yes - a: ~ - a: - b: nope ``` then ```bash yq '.[] | has("a")' sample.yml ``` will output ```yaml true true true false ``` ## Select, checking for existence of deep paths Simply pipe in parent expressions into `has` Given a sample.yml file of: ```yaml - a: b: c: cat - a: b: d: dog ``` then ```bash yq '.[] | select(.a.b | has("c"))' sample.yml ``` will output ```yaml a: b: c: cat ``` ## Has array index Given a sample.yml file of: ```yaml - [] - [1] - [1, 2] - [1, null] - [1, 2, 3] ``` then ```bash yq '.[] | has(1)' sample.yml ``` will output ```yaml false false true true true ``` ================================================ FILE: pkg/yqlib/doc/operators/headers/Main.md ================================================ # NAME *yq* is a portable command-line data file processor # SYNOPSIS yq [eval/eval-all] [expression] files.. eval/e - (default) Apply the expression to each document in each yaml file in sequence eval-all/ea - Loads all yaml documents of all yaml files and runs expression once # DESCRIPTION a lightweight and portable command-line data file processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously. This documentation is also available at https://mikefarah.gitbook.io/yq/ # QUICK GUIDE ## Read a value: ```bash yq '.a.b[0].c' file.yaml ``` ## Pipe from STDIN: ```bash cat file.yaml | yq '.a.b[0].c' ``` ## Update a yaml file, in place ```bash yq -i '.a.b[0].c = "cool"' file.yaml ``` ## Update using environment variables ```bash NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml ``` ## Merge multiple files ``` yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml ``` Note the use of `ea` to evaluate all files at once (instead of in sequence.) ## Multiple updates to a yaml file ```bash yq -i ' .a.b[0].c = "cool" | .x.y.z = "foobar" | .person.name = strenv(NAME) ' file.yaml ``` See the [documentation](https://mikefarah.gitbook.io/yq/) for more. # KNOWN ISSUES / MISSING FEATURES - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details) - Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell # BUGS / ISSUES / FEATURE REQUESTS Please visit the GitHub page https://github.com/mikefarah/yq/. ================================================ FILE: pkg/yqlib/doc/operators/headers/add.md ================================================ # Add Add behaves differently according to the type of the LHS: * arrays: concatenate * number scalars: arithmetic addition * string scalars: concatenate * maps: shallow merge (use the multiply operator (`*`) to deeply merge) Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. ================================================ FILE: pkg/yqlib/doc/operators/headers/alternative-default-value.md ================================================ # Alternative (Default value) This operator is used to provide alternative (or default) values when a particular expression is either null or false. ================================================ FILE: pkg/yqlib/doc/operators/headers/anchor-and-alias-operators.md ================================================ # Anchor and Alias Operators Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names). `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag. ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed). This flag also enables advanced merging, like inline maps, as well as fixes to ensure when exploding a particular path, neighbours are not affect ed. Long story short, you should be setting this flag to true. See examples of the flag differences below, where LEGACY is with the flag off; and FIXED is with the flag on. ================================================ FILE: pkg/yqlib/doc/operators/headers/array-to-map.md ================================================ # Array to Map Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over. Behind the scenes, this is implemented using reduce: ``` (.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i) ``` ================================================ FILE: pkg/yqlib/doc/operators/headers/assign-update.md ================================================ # Assign (Update) This operator is used to update node values. It can be used in either the: ### plain form: `=` Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline. ### relative form: `|=` This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment. ### Flags - `c` clobber custom tags ================================================ FILE: pkg/yqlib/doc/operators/headers/boolean-operators.md ================================================ # Boolean Operators The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. `any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true. `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet These are most commonly used with the `select` operator to filter particular nodes. ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ================================================ FILE: pkg/yqlib/doc/operators/headers/collect-into-array.md ================================================ # Collect into Array This creates an array using the expression between the square brackets. ================================================ FILE: pkg/yqlib/doc/operators/headers/column.md ================================================ # Column Returns the column of the matching node. Starts from 1, 0 indicates there was no column data. Column is the number of characters that precede that node on the line it starts. ================================================ FILE: pkg/yqlib/doc/operators/headers/comment-operators.md ================================================ # Comment Operators Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples. Like the `=` and `|=` assign operators, the same syntax applies when updating comments: ### plain form: `=` This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline ### relative form: `|=` This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path. ================================================ FILE: pkg/yqlib/doc/operators/headers/compare.md ================================================ # Compare Operators Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time. The following types are currently supported: - numbers - strings - datetimes ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ================================================ FILE: pkg/yqlib/doc/operators/headers/contains.md ================================================ # Contains This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring. {% hint style="warning" %} _Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains(["cat"])` will return true for an array `["cats"]`. See the "Array has a subset array" example below on how to check for a subset. {% endhint %} ================================================ FILE: pkg/yqlib/doc/operators/headers/create-collect-into-object.md ================================================ # Create, Collect into Object This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents. ================================================ FILE: pkg/yqlib/doc/operators/headers/datetime.md ================================================ # Date Time Various operators for parsing and manipulating dates. ## Date time formatting This uses Golang's built in time library for parsing and formatting date times. When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing. To specify a custom parsing format, use the `with_dtf` operator. The first parameter sets the datetime parsing format for the expression in the second parameter. The expression can be any valid `yq` expression tree. ```bash yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))' ``` See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options. ## Timezones This uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details. ## Durations Durations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function. You can add durations to time using the `+` operator. ================================================ FILE: pkg/yqlib/doc/operators/headers/delete.md ================================================ # Delete Deletes matching entries in maps or arrays. ================================================ FILE: pkg/yqlib/doc/operators/headers/divide.md ================================================ # Divide Divide behaves differently according to the type of the LHS: * strings: split by the divider * number: arithmetic division ================================================ FILE: pkg/yqlib/doc/operators/headers/document-index.md ================================================ # Document Index Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document. ================================================ FILE: pkg/yqlib/doc/operators/headers/encode-decode.md ================================================ # Encoder / Decoder Encode operators will take the piped in object structure and encode it as a string in the desired format. The decode operators do the opposite, they take a formatted string and decode it into the relevant object structure. Note that you can optionally pass an indent value to the encode functions (see below). These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them. | Format | Decode (from string) | Encode (to string) | | --- | -- | --| | Yaml | from_yaml/@yamld | to_yaml(i)/@yaml | | JSON | from_json/@jsond | to_json(i)/@json | | Properties | from_props/@propsd | to_props/@props | | CSV | from_csv/@csvd | to_csv/@csv | | TSV | from_tsv/@tsvd | to_tsv/@tsv | | XML | from_xml/@xmld | to_xml(i)/@xml | | Base64 | @base64d | @base64 | | URI | @urid | @uri | | Shell | | @sh | See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats. XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields. Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content. ================================================ FILE: pkg/yqlib/doc/operators/headers/entries.md ================================================ # Entries Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. Use `with_entries(op)` as a syntactic sugar for doing `to_entries | op | from_entries`. ================================================ FILE: pkg/yqlib/doc/operators/headers/env-variable-operators.md ================================================ # Env Variable Operators These operators are used to handle environment variables usage in expressions and documents. While environment variables can, of course, be passed in via your CLI with string interpolation, this often comes with complex quote escaping and can be tricky to write and read. There are three operators: - `env` which takes a single environment variable name and parse the variable as a yaml node (be it a map, array, string, number of boolean) - `strenv` which also takes a single environment variable name, and always parses the variable as a string. - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). ## EnvSubst Options You can optionally pass envsubst any of the following options: - nu: NoUnset, this will fail if there are any referenced variables that are not set - ne: NoEmpty, this will fail if there are any referenced variables that are empty - ff: FailFast, this will abort on the first failure (rather than collect all the errors) E.g: `envsubst(ne, ff)` will fail on the first empty variable. See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. ## Tip To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator as follows: ```bash yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml ``` ## Disabling env operators If required, you can use the `--security-disable-env-ops` to disable env operations. ================================================ FILE: pkg/yqlib/doc/operators/headers/equals.md ================================================ # Equals / Not Equals This is a boolean operator that will return `true` if the LHS is equal to the RHS and `false` otherwise. ``` .a == .b ``` It is most often used with the select operator to find particular nodes: ``` select(.a == .b) ``` The not equals `!=` operator returns `false` if the LHS is equal to the RHS. ## Related Operators - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) - select operator [here](https://mikefarah.gitbook.io/yq/operators/select) ================================================ FILE: pkg/yqlib/doc/operators/headers/error.md ================================================ # Error Use this operation to short-circuit expressions. Useful for validation. ================================================ FILE: pkg/yqlib/doc/operators/headers/eval.md ================================================ # Eval Use `eval` to dynamically process an expression - for instance from an environment variable. `eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. Tip: This can be a useful way to parameterise complex scripts. ================================================ FILE: pkg/yqlib/doc/operators/headers/file-operators.md ================================================ # File Operators File operators are most often used with merge when needing to merge specific files together. Note that when doing this, you will need to use `eval-all` to ensure all yaml documents are loaded into memory before performing the merge (as opposed to `eval` which runs the expression once per document). Note that the `fileIndex` operator has a short alias of `fi`. ## Merging files Note the use of eval-all to ensure all documents are loaded into memory. ```bash yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml ``` ================================================ FILE: pkg/yqlib/doc/operators/headers/filter.md ================================================ # Filter Filters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`. ================================================ FILE: pkg/yqlib/doc/operators/headers/first.md ================================================ # First Returns the first matching element in an array, or first matching value in a map. Can be given an expression to match with, otherwise will just return the first. ================================================ FILE: pkg/yqlib/doc/operators/headers/flatten.md ================================================ # Flatten This recursively flattens arrays. ================================================ FILE: pkg/yqlib/doc/operators/headers/group-by.md ================================================ # Group By This is used to group items in an array by an expression. ================================================ FILE: pkg/yqlib/doc/operators/headers/has.md ================================================ # Has This operation returns true if the key exists in a map (or index in an array), false otherwise. ================================================ FILE: pkg/yqlib/doc/operators/headers/keys.md ================================================ # Keys Use the `keys` operator to return map keys or array indices. ================================================ FILE: pkg/yqlib/doc/operators/headers/kind.md ================================================ # Kind The `kind` operator identifies the type of a node as either `scalar`, `map`, or `seq`. This can be used for filtering or transforming nodes based on their type. Note that `null` values are treated as `scalar`. ================================================ FILE: pkg/yqlib/doc/operators/headers/length.md ================================================ # Length Returns the lengths of the nodes. Length is defined according to the type of the node. ================================================ FILE: pkg/yqlib/doc/operators/headers/line.md ================================================ # Line Returns the line of the matching node. Starts from 1, 0 indicates there was no line data. ================================================ FILE: pkg/yqlib/doc/operators/headers/load.md ================================================ # Load The load operators allows you to load in content from another file. Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. You can load files of the following supported types: |Format | Load Operator | | --- | --- | | Yaml | load | | XML | load_xml | | Properties | load_props | | Plain String | load_str | | Base64 | load_base64 | Note that load_base64 only works for base64 encoded utf-8 strings. ## Samples files for tests: ### yaml `../../examples/thing.yml`: ```yaml a: apple is included b: cool ``` ### xml `small.xml`: ```xml is some xml ``` ### properties `small.properties`: ```properties this.is = a properties file ``` ### base64 `base64.txt`: ``` bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u ``` ## Disabling file operators If required, you can use the `--security-disable-file-ops` to disable file operations. ================================================ FILE: pkg/yqlib/doc/operators/headers/map.md ================================================ # Map Maps values of an array. Use `map_values` to map values of an object. ================================================ FILE: pkg/yqlib/doc/operators/headers/max.md ================================================ # Max Computes the maximum among an incoming sequence of scalar values. ================================================ FILE: pkg/yqlib/doc/operators/headers/min.md ================================================ # Min Computes the minimum among an incoming sequence of scalar values. ================================================ FILE: pkg/yqlib/doc/operators/headers/modulo.md ================================================ # Modulo Arithmetic modulo operator, returns the remainder from dividing two numbers. ================================================ FILE: pkg/yqlib/doc/operators/headers/multiply-merge.md ================================================ # Multiply (Merge) Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. ## Objects and arrays - merging Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged. You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add). Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. ### Merge Flags You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below - `+` append arrays - `d` deeply merge arrays - `?` only merge _existing_ fields - `n` only merge _new_ fields - `c` clobber custom tags To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add). ### Merge two files together This uses the load operator to merge file2 into file1. ```bash yq '. *= load("file2.yml")' file1.yml ``` ### Merging all files Note the use of `eval-all` to ensure all documents are loaded into memory. ```bash yq eval-all '. as $item ireduce ({}; . * $item )' *.yml ``` # Merging complex arrays together by a key field By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array. For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key) ================================================ FILE: pkg/yqlib/doc/operators/headers/omit.md ================================================ # Omit Works like `pick`, but instead you specify the keys/indices that you _don't_ want included. ================================================ FILE: pkg/yqlib/doc/operators/headers/parent.md ================================================ # Parent Parent simply returns the parent nodes of the matching nodes. ================================================ FILE: pkg/yqlib/doc/operators/headers/path.md ================================================ # Path The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node. You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key. Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays. ================================================ FILE: pkg/yqlib/doc/operators/headers/pick.md ================================================ # Pick Filter a map by the specified list of keys. Map is returned with the key in the order of the pick list. Similarly, filter an array by the specified list of indices. ================================================ FILE: pkg/yqlib/doc/operators/headers/pipe.md ================================================ # Pipe Pipe the results of an expression into another. Like the bash operator. ================================================ FILE: pkg/yqlib/doc/operators/headers/pivot.md ================================================ # Pivot Emulates the `PIVOT` function supported by several popular RDBMS systems. ================================================ FILE: pkg/yqlib/doc/operators/headers/recursive-descent-glob.md ================================================ # Recursive Descent (Glob) This operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. ## match values form `..` This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values. For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys: ```bash yq '.. style= "flow"' file.yaml ``` ## match values and map keys form `...` The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling and tags and also use anchors and aliases. For instance to set the `style` of all nodes in a yaml doc, including the map keys: ```bash yq '... style= "flow"' file.yaml ``` ================================================ FILE: pkg/yqlib/doc/operators/headers/reduce.md ================================================ # Reduce Reduce is a powerful way to process a collection of data into a new form. ``` as $ ireduce (; ) ``` e.g. ``` .[] as $item ireduce (0; . + $item) ``` On the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements. On the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. ## yq vs jq syntax Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`). To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added. ================================================ FILE: pkg/yqlib/doc/operators/headers/reverse.md ================================================ # Reverse Reverses the order of the items in an array ================================================ FILE: pkg/yqlib/doc/operators/headers/select.md ================================================ # Select Select is used to filter arrays and maps by a boolean expression. ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) ================================================ FILE: pkg/yqlib/doc/operators/headers/shuffle.md ================================================ # Shuffle Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order. ================================================ FILE: pkg/yqlib/doc/operators/headers/slice-array.md ================================================ # Slice/Splice Array The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. You may leave out the first or second number, which will refer to the start or end of the array respectively. ================================================ FILE: pkg/yqlib/doc/operators/headers/sort-keys.md ================================================ # Sort Keys The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). Sort is particularly useful for diffing two different yaml documents: ```bash yq -i -P 'sort_keys(..)' file1.yml yq -i -P 'sort_keys(..)' file2.yml diff file1.yml file2.yml ``` Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors. For more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`. ================================================ FILE: pkg/yqlib/doc/operators/headers/sort.md ================================================ # Sort Sorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield). To sort by descending order, pipe the results through the `reverse` operator after sorting. Note that at this stage, `yq` only sorts scalar fields. ================================================ FILE: pkg/yqlib/doc/operators/headers/split-into-documents.md ================================================ # Split into Documents This operator splits all matches into separate documents ================================================ FILE: pkg/yqlib/doc/operators/headers/string-operators.md ================================================ # String Operators ## RegEx This uses Golang's native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax. Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats")`. ### match(regEx) This operator returns the substring match details of the given regEx. ### capture(regEx) Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing. ## test(regEx) Returns true if the string matches the RegEx, false otherwise. ## sub(regEx, replacement) Substitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second parameter specifies what to replace those matches with. This can refer to capture groups from the first RegEx. ## String blocks, bash and newlines Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_. For instance to get this yaml: ``` a: | cat ``` Using `$( exp )` wont work, as it will trim the trailing newline. ``` m=$(echo "cat\n") yq -n '.a = strenv(m)' a: cat ``` However, using printf works: ``` printf -v m "cat\n" ; m="$m" yq -n '.a = strenv(m)' a: | cat ``` As well as having multiline expressions: ``` m="cat " yq -n '.a = strenv(m)' a: | cat ``` Similarly, if you're trying to set the content from a file, and want a trailing newline: ``` IFS= read -rd '' output < <(cat my_file) output=$output ./yq '.data.values = strenv(output)' first.yml ``` ================================================ FILE: pkg/yqlib/doc/operators/headers/style.md ================================================ # Style The style operator can be used to get or set the style of nodes (e.g. string style, yaml style). Use this to control the formatting of the document in yaml. ================================================ FILE: pkg/yqlib/doc/operators/headers/subtract.md ================================================ # Subtract You can use subtract to subtract numbers as well as remove elements from an array. ================================================ FILE: pkg/yqlib/doc/operators/headers/tag.md ================================================ # Tag The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`). ================================================ FILE: pkg/yqlib/doc/operators/headers/to_number.md ================================================ # To Number Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. ================================================ FILE: pkg/yqlib/doc/operators/headers/traverse-read.md ================================================ # Traverse (Read) This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) See examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on. ================================================ FILE: pkg/yqlib/doc/operators/headers/union.md ================================================ # Union This operator is used to combine different results together. ================================================ FILE: pkg/yqlib/doc/operators/headers/unique.md ================================================ # Unique This is used to filter out duplicated items in an array. Note that the original order of the array is maintained. ================================================ FILE: pkg/yqlib/doc/operators/headers/variable-operators.md ================================================ # Variable Operators Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. ================================================ FILE: pkg/yqlib/doc/operators/headers/with.md ================================================ # With Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context. ================================================ FILE: pkg/yqlib/doc/operators/keys.md ================================================ # Keys Use the `keys` operator to return map keys or array indices. ## Map keys Given a sample.yml file of: ```yaml dog: woof cat: meow ``` then ```bash yq 'keys' sample.yml ``` will output ```yaml - dog - cat ``` ## Array keys Given a sample.yml file of: ```yaml - apple - banana ``` then ```bash yq 'keys' sample.yml ``` will output ```yaml - 0 - 1 ``` ## Retrieve array key Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq '.[1] | key' sample.yml ``` will output ```yaml 1 ``` ## Retrieve map key Given a sample.yml file of: ```yaml a: thing ``` then ```bash yq '.a | key' sample.yml ``` will output ```yaml a ``` ## No key Given a sample.yml file of: ```yaml {} ``` then ```bash yq 'key' sample.yml ``` will output ```yaml ``` ## Update map key Given a sample.yml file of: ```yaml a: x: 3 y: 4 ``` then ```bash yq '(.a.x | key) = "meow"' sample.yml ``` will output ```yaml a: meow: 3 y: 4 ``` ## Get comment from map key Given a sample.yml file of: ```yaml a: # comment on key x: 3 y: 4 ``` then ```bash yq '.a.x | key | headComment' sample.yml ``` will output ```yaml comment on key ``` ## Check node is a key Given a sample.yml file of: ```yaml a: b: - cat c: frog ``` then ```bash yq '[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]' sample.yml ``` will output ```yaml - p: "" isKey: false tag: '!!map' - p: a isKey: true tag: '!!str' - p: a isKey: false tag: '!!map' - p: a.b isKey: true tag: '!!str' - p: a.b isKey: false tag: '!!seq' - p: a.b.0 isKey: false tag: '!!str' - p: a.c isKey: true tag: '!!str' - p: a.c isKey: false tag: '!!str' ``` ================================================ FILE: pkg/yqlib/doc/operators/kind.md ================================================ # Kind The `kind` operator identifies the type of a node as either `scalar`, `map`, or `seq`. This can be used for filtering or transforming nodes based on their type. Note that `null` values are treated as `scalar`. ## Get kind Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: [] g: {} h: null ``` then ```bash yq '.. | kind' sample.yml ``` will output ```yaml map scalar scalar scalar scalar seq map scalar ``` ## Get kind, ignores custom tags Unlike tag, kind is not affected by custom tags. Given a sample.yml file of: ```yaml a: !!thing cat b: !!foo {} c: !!bar [] ``` then ```bash yq '.. | kind' sample.yml ``` will output ```yaml map scalar map seq ``` ## Add comments only to scalars An example of how you can use kind Given a sample.yml file of: ```yaml a: b: 5 c: 3.2 e: true f: [] g: {} h: null ``` then ```bash yq '(.. | select(kind == "scalar")) line_comment = "this is a scalar"' sample.yml ``` will output ```yaml a: b: 5 # this is a scalar c: 3.2 # this is a scalar e: true # this is a scalar f: [] g: {} h: null # this is a scalar ``` ================================================ FILE: pkg/yqlib/doc/operators/length.md ================================================ # Length Returns the lengths of the nodes. Length is defined according to the type of the node. ## String length returns length of string Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a | length' sample.yml ``` will output ```yaml 3 ``` ## null length Given a sample.yml file of: ```yaml a: null ``` then ```bash yq '.a | length' sample.yml ``` will output ```yaml 0 ``` ## Map length returns number of entries Given a sample.yml file of: ```yaml a: cat c: dog ``` then ```bash yq 'length' sample.yml ``` will output ```yaml 2 ``` ## Array length returns number of elements Given a sample.yml file of: ```yaml - 2 - 4 - 6 - 8 ``` then ```bash yq 'length' sample.yml ``` will output ```yaml 4 ``` ================================================ FILE: pkg/yqlib/doc/operators/line.md ================================================ # Line Returns the line of the matching node. Starts from 1, 0 indicates there was no line data. ## Returns line of _value_ node Given a sample.yml file of: ```yaml a: cat b: c: cat ``` then ```bash yq '.b | line' sample.yml ``` will output ```yaml 3 ``` ## Returns line of _key_ node Pipe through the key operator to get the line of the key Given a sample.yml file of: ```yaml a: cat b: c: cat ``` then ```bash yq '.b | key | line' sample.yml ``` will output ```yaml 2 ``` ## First line is 1 Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a | line' sample.yml ``` will output ```yaml 1 ``` ## No line data is 0 Running ```bash yq --null-input '{"a": "new entry"} | line' ``` will output ```yaml 0 ``` ================================================ FILE: pkg/yqlib/doc/operators/load.md ================================================ # Load The load operators allows you to load in content from another file. Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system. You can load files of the following supported types: |Format | Load Operator | | --- | --- | | Yaml | load | | XML | load_xml | | Properties | load_props | | Plain String | load_str | | Base64 | load_base64 | Note that load_base64 only works for base64 encoded utf-8 strings. ## Samples files for tests: ### yaml `../../examples/thing.yml`: ```yaml a: apple is included b: cool ``` ### xml `small.xml`: ```xml is some xml ``` ### properties `small.properties`: ```properties this.is = a properties file ``` ### base64 `base64.txt`: ``` bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u ``` ## Disabling file operators If required, you can use the `--security-disable-file-ops` to disable file operations. ## Simple example Given a sample.yml file of: ```yaml myFile: ../../examples/thing.yml ``` then ```bash yq 'load(.myFile)' sample.yml ``` will output ```yaml a: apple is included b: cool. ``` ## Replace node with referenced file Note that you can modify the filename in the load operator if needed. Given a sample.yml file of: ```yaml something: file: thing.yml ``` then ```bash yq '.something |= load("../../examples/" + .file)' sample.yml ``` will output ```yaml something: a: apple is included b: cool. ``` ## Replace _all_ nodes with referenced file Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. Given a sample.yml file of: ```yaml something: file: thing.yml over: here: - file: thing.yml ``` then ```bash yq '(.. | select(has("file"))) |= load("../../examples/" + .file)' sample.yml ``` will output ```yaml something: a: apple is included b: cool. over: here: - a: apple is included b: cool. ``` ## Replace node with referenced file as string This will work for any text based file Given a sample.yml file of: ```yaml something: file: thing.yml ``` then ```bash yq '.something |= load_str("../../examples/" + .file)' sample.yml ``` will output ```yaml something: |- a: apple is included b: cool. ``` ## Load from XML Given a sample.yml file of: ```yaml cool: things ``` then ```bash yq '.more_stuff = load_xml("../../examples/small.xml")' sample.yml ``` will output ```yaml cool: things more_stuff: this: is some xml ``` ## Load from Properties Given a sample.yml file of: ```yaml cool: things ``` then ```bash yq '.more_stuff = load_props("../../examples/small.properties")' sample.yml ``` will output ```yaml cool: things more_stuff: this: is: a properties file ``` ## Merge from properties This can be used as a convenient way to update a yaml document Given a sample.yml file of: ```yaml this: is: from yaml cool: ay ``` then ```bash yq '. *= load_props("../../examples/small.properties")' sample.yml ``` will output ```yaml this: is: a properties file cool: ay ``` ## Load from base64 encoded file Given a sample.yml file of: ```yaml cool: things ``` then ```bash yq '.more_stuff = load_base64("../../examples/base64.txt")' sample.yml ``` will output ```yaml cool: things more_stuff: my secret chilli recipe is.... ``` ## load() operation fails when security is enabled Use `--security-disable-file-ops` to disable file operations for security. Running ```bash yq --null-input 'load("../../examples/thing.yml")' ``` will output ```bash Error: file operations have been disabled ``` ## load_str() operation fails when security is enabled Use `--security-disable-file-ops` to disable file operations for security. Running ```bash yq --null-input 'load_str("../../examples/thing.yml")' ``` will output ```bash Error: file operations have been disabled ``` ## load_xml() operation fails when security is enabled Use `--security-disable-file-ops` to disable file operations for security. Running ```bash yq --null-input 'load_xml("../../examples/small.xml")' ``` will output ```bash Error: file operations have been disabled ``` ## load_props() operation fails when security is enabled Use `--security-disable-file-ops` to disable file operations for security. Running ```bash yq --null-input 'load_props("../../examples/small.properties")' ``` will output ```bash Error: file operations have been disabled ``` ## load_base64() operation fails when security is enabled Use `--security-disable-file-ops` to disable file operations for security. Running ```bash yq --null-input 'load_base64("../../examples/base64.txt")' ``` will output ```bash Error: file operations have been disabled ``` ================================================ FILE: pkg/yqlib/doc/operators/map.md ================================================ # Map Maps values of an array. Use `map_values` to map values of an object. ## Map array Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq 'map(. + 1)' sample.yml ``` will output ```yaml - 2 - 3 - 4 ``` ## Map object values Given a sample.yml file of: ```yaml a: 1 b: 2 c: 3 ``` then ```bash yq 'map_values(. + 1)' sample.yml ``` will output ```yaml a: 2 b: 3 c: 4 ``` ================================================ FILE: pkg/yqlib/doc/operators/max.md ================================================ # Max Computes the maximum among an incoming sequence of scalar values. ## Maximum int Given a sample.yml file of: ```yaml - 99 - 16 - 12 - 6 - 66 ``` then ```bash yq 'max' sample.yml ``` will output ```yaml 99 ``` ## Maximum string Given a sample.yml file of: ```yaml - foo - bar - baz ``` then ```bash yq 'max' sample.yml ``` will output ```yaml foo ``` ## Maximum of empty Given a sample.yml file of: ```yaml [] ``` then ```bash yq 'max' sample.yml ``` will output ```yaml ``` ================================================ FILE: pkg/yqlib/doc/operators/min.md ================================================ # Min Computes the minimum among an incoming sequence of scalar values. ## Minimum int Given a sample.yml file of: ```yaml - 99 - 16 - 12 - 6 - 66 ``` then ```bash yq 'min' sample.yml ``` will output ```yaml 6 ``` ## Minimum string Given a sample.yml file of: ```yaml - foo - bar - baz ``` then ```bash yq 'min' sample.yml ``` will output ```yaml bar ``` ## Minimum of empty Given a sample.yml file of: ```yaml [] ``` then ```bash yq 'min' sample.yml ``` will output ```yaml ``` ================================================ FILE: pkg/yqlib/doc/operators/modulo.md ================================================ # Modulo Arithmetic modulo operator, returns the remainder from dividing two numbers. ## Number modulo - int If the lhs and rhs are ints then the expression will be calculated with ints. Given a sample.yml file of: ```yaml a: 13 b: 2 ``` then ```bash yq '.a = .a % .b' sample.yml ``` will output ```yaml a: 1 b: 2 ``` ## Number modulo - float If the lhs or rhs are floats then the expression will be calculated with floats. Given a sample.yml file of: ```yaml a: 12 b: 2.5 ``` then ```bash yq '.a = .a % .b' sample.yml ``` will output ```yaml a: !!float 2 b: 2.5 ``` ## Number modulo - int by zero If the lhs is an int and rhs is a 0 the result is an error. Given a sample.yml file of: ```yaml a: 1 b: 0 ``` then ```bash yq '.a = .a % .b' sample.yml ``` will output ```bash Error: cannot modulo by 0 ``` ## Number modulo - float by zero If the lhs is a float and rhs is a 0 the result is NaN. Given a sample.yml file of: ```yaml a: 1.1 b: 0 ``` then ```bash yq '.a = .a % .b' sample.yml ``` will output ```yaml a: !!float NaN b: 0 ``` ================================================ FILE: pkg/yqlib/doc/operators/multiply-merge.md ================================================ # Multiply (Merge) Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. ## Objects and arrays - merging Objects are merged _deeply_ matching on matching keys. By default, array values override and are not deeply merged. You can use the add operator `+`, to shallow merge objects, see more info [here](https://mikefarah.gitbook.io/yq/operators/add). Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. ### Merge Flags You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below - `+` append arrays - `d` deeply merge arrays - `?` only merge _existing_ fields - `n` only merge _new_ fields - `c` clobber custom tags To perform a shallow merge only, use the add operator `+`, see more info [here](https://mikefarah.gitbook.io/yq/operators/add). ### Merge two files together This uses the load operator to merge file2 into file1. ```bash yq '. *= load("file2.yml")' file1.yml ``` ### Merging all files Note the use of `eval-all` to ensure all documents are loaded into memory. ```bash yq eval-all '. as $item ireduce ({}; . * $item )' *.yml ``` # Merging complex arrays together by a key field By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array. For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key) ## Multiply integers Given a sample.yml file of: ```yaml a: 3 b: 4 ``` then ```bash yq '.a *= .b' sample.yml ``` will output ```yaml a: 12 b: 4 ``` ## Multiply string node X int Given a sample.yml file of: ```yaml b: banana ``` then ```bash yq '.b * 4' sample.yml ``` will output ```yaml bananabananabananabanana ``` ## Multiply int X string node Given a sample.yml file of: ```yaml b: banana ``` then ```bash yq '4 * .b' sample.yml ``` will output ```yaml bananabananabananabanana ``` ## Multiply string X int node Given a sample.yml file of: ```yaml n: 4 ``` then ```bash yq '"banana" * .n' sample.yml ``` will output ```yaml bananabananabananabanana ``` ## Multiply int node X string Given a sample.yml file of: ```yaml n: 4 ``` then ```bash yq '.n * "banana"' sample.yml ``` will output ```yaml bananabananabananabanana ``` ## Merge objects together, returning merged result only Given a sample.yml file of: ```yaml a: field: me fieldA: cat b: field: g: wizz fieldB: dog ``` then ```bash yq '.a * .b' sample.yml ``` will output ```yaml field: g: wizz fieldA: cat fieldB: dog ``` ## Merge objects together, returning parent object Given a sample.yml file of: ```yaml a: field: me fieldA: cat b: field: g: wizz fieldB: dog ``` then ```bash yq '. * {"a":.b}' sample.yml ``` will output ```yaml a: field: g: wizz fieldA: cat fieldB: dog b: field: g: wizz fieldB: dog ``` ## Merge keeps style of LHS Given a sample.yml file of: ```yaml a: {things: great} b: also: "me" ``` then ```bash yq '. * {"a":.b}' sample.yml ``` will output ```yaml a: {things: great, also: "me"} b: also: "me" ``` ## Merge arrays Given a sample.yml file of: ```yaml a: - 1 - 2 - 3 b: - 3 - 4 - 5 ``` then ```bash yq '. * {"a":.b}' sample.yml ``` will output ```yaml a: - 3 - 4 - 5 b: - 3 - 4 - 5 ``` ## Merge, only existing fields Given a sample.yml file of: ```yaml a: thing: one cat: frog b: missing: two thing: two ``` then ```bash yq '.a *? .b' sample.yml ``` will output ```yaml thing: two cat: frog ``` ## Merge, only new fields Given a sample.yml file of: ```yaml a: thing: one cat: frog b: missing: two thing: two ``` then ```bash yq '.a *n .b' sample.yml ``` will output ```yaml thing: one cat: frog missing: two ``` ## Merge, appending arrays Given a sample.yml file of: ```yaml a: array: - 1 - 2 - animal: dog value: coconut b: array: - 3 - 4 - animal: cat value: banana ``` then ```bash yq '.a *+ .b' sample.yml ``` will output ```yaml array: - 1 - 2 - animal: dog - 3 - 4 - animal: cat value: banana ``` ## Merge, only existing fields, appending arrays Given a sample.yml file of: ```yaml a: thing: - 1 - 2 b: thing: - 3 - 4 another: - 1 ``` then ```bash yq '.a *?+ .b' sample.yml ``` will output ```yaml thing: - 1 - 2 - 3 - 4 ``` ## Merge, deeply merging arrays Merging arrays deeply means arrays are merged like objects, with indices as their key. In this case, we merge the first item in the array and do nothing with the second. Given a sample.yml file of: ```yaml a: - name: fred age: 12 - name: bob age: 32 b: - name: fred age: 34 ``` then ```bash yq '.a *d .b' sample.yml ``` will output ```yaml - name: fred age: 34 - name: bob age: 32 ``` ## Merge arrays of objects together, matching on a key This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below. It merges in the array provided in the second file into the first - matching on equal keys. Explanation: The approach, at a high level, is to reduce into a merged map (keyed by the unique key) and then convert that back into an array. First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by. The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. Next, we convert the map back to an array, using reduce again, concatenating all the map values together. Finally, we set the result of the merged array back into the first doc. Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) Given a sample.yml file of: ```yaml myArray: - a: apple b: appleB - a: kiwi b: kiwiB - a: banana b: bananaB something: else ``` And another sample another.yml file of: ```yaml newArray: - a: banana c: bananaC - a: apple b: appleB2 - a: dingo c: dingoC ``` then ```bash idPath=".a" originalPath=".myArray" otherPath=".newArray" yq eval-all ' ( (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) ) as $mergedArray | select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray ' sample.yml another.yml ``` will output ```yaml myArray: - a: apple b: appleB2 - a: kiwi b: kiwiB - a: banana b: bananaB c: bananaC - a: dingo c: dingoC something: else ``` ## Merge to prefix an element Given a sample.yml file of: ```yaml a: cat b: dog ``` then ```bash yq '. * {"a": {"c": .a}}' sample.yml ``` will output ```yaml a: c: cat b: dog ``` ## Merge with simple aliases Given a sample.yml file of: ```yaml a: &cat c: frog b: f: *cat c: g: thongs ``` then ```bash yq '.c * .b' sample.yml ``` will output ```yaml g: thongs f: *cat ``` ## Merge copies anchor names Given a sample.yml file of: ```yaml a: c: &cat frog b: f: *cat c: g: thongs ``` then ```bash yq '.c * .a' sample.yml ``` will output ```yaml g: thongs c: &cat frog ``` ## Merge with merge anchors Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar * .foobarList' sample.yml ``` will output ```yaml c: foobarList_c !!merge <<: - *foo - *bar thing: foobar_thing b: foobarList_b ``` ## Custom types: that are really numbers When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml a: !horse 2 b: !goat 3 ``` then ```bash yq '.a = .a * .b' sample.yml ``` will output ```yaml a: !horse 6 b: !goat 3 ``` ## Custom types: that are really maps Custom tags will be maintained. Given a sample.yml file of: ```yaml a: !horse cat: meow b: !goat dog: woof ``` then ```bash yq '.a = .a * .b' sample.yml ``` will output ```yaml a: !horse cat: meow dog: woof b: !goat dog: woof ``` ## Custom types: clobber tags Use the `c` option to clobber custom tags. Note that the second tag is now used. Given a sample.yml file of: ```yaml a: !horse cat: meow b: !goat dog: woof ``` then ```bash yq '.a *=c .b' sample.yml ``` will output ```yaml a: !goat cat: meow dog: woof b: !goat dog: woof ``` ## Merging a null with a map Running ```bash yq --null-input 'null * {"some": "thing"}' ``` will output ```yaml some: thing ``` ## Merging a map with null Running ```bash yq --null-input '{"some": "thing"} * null' ``` will output ```yaml some: thing ``` ## Merging a null with an array Running ```bash yq --null-input 'null * ["some"]' ``` will output ```yaml - some ``` ## Merging an array with null Running ```bash yq --null-input '["some"] * null' ``` will output ```yaml - some ``` ================================================ FILE: pkg/yqlib/doc/operators/omit.md ================================================ # Omit Works like `pick`, but instead you specify the keys/indices that you _don't_ want included. ## Omit keys from map Note that non existent keys are skipped. Given a sample.yml file of: ```yaml myMap: cat: meow dog: bark thing: hamster hamster: squeak ``` then ```bash yq '.myMap |= omit(["hamster", "cat", "goat"])' sample.yml ``` will output ```yaml myMap: dog: bark thing: hamster ``` ## Omit indices from array Note that non existent indices are skipped. Given a sample.yml file of: ```yaml - cat - leopard - lion ``` then ```bash yq 'omit([2, 0, 734, -5])' sample.yml ``` will output ```yaml - leopard ``` ================================================ FILE: pkg/yqlib/doc/operators/parent.md ================================================ # Parent Parent simply returns the parent nodes of the matching nodes. ## Simple example Given a sample.yml file of: ```yaml a: nested: cat ``` then ```bash yq '.a.nested | parent' sample.yml ``` will output ```yaml nested: cat ``` ## Parent of nested matches Given a sample.yml file of: ```yaml a: fruit: apple name: bob b: fruit: banana name: sam ``` then ```bash yq '.. | select(. == "banana") | parent' sample.yml ``` will output ```yaml fruit: banana name: sam ``` ## Get parent attribute Given a sample.yml file of: ```yaml a: fruit: apple name: bob b: fruit: banana name: sam ``` then ```bash yq '.. | select(. == "banana") | parent.name' sample.yml ``` will output ```yaml sam ``` ## Get parents Match all parents Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | parents' sample.yml ``` will output ```yaml - c: cat - b: c: cat - a: b: c: cat ``` ## Get the top (root) parent Use negative numbers to get the top parents. You can think of this as indexing into the 'parents' array above Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | parent(-1)' sample.yml ``` will output ```yaml a: b: c: cat ``` ## Root Alias for parent(-1), returns the top level parent. This is usually the document node. Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | root' sample.yml ``` will output ```yaml a: b: c: cat ``` ## N-th parent You can optionally supply the number of levels to go up for the parent, the default being 1. Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | parent(2)' sample.yml ``` will output ```yaml b: c: cat ``` ## N-th parent - another level Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | parent(3)' sample.yml ``` will output ```yaml a: b: c: cat ``` ## N-th negative Similarly, use negative numbers to index backwards from the parents array Given a sample.yml file of: ```yaml a: b: c: cat ``` then ```bash yq '.a.b.c | parent(-2)' sample.yml ``` will output ```yaml b: c: cat ``` ## No parent Given a sample.yml file of: ```yaml {} ``` then ```bash yq 'parent' sample.yml ``` will output ```yaml ``` ================================================ FILE: pkg/yqlib/doc/operators/path.md ================================================ # Path The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node. You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key. Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays. ## Map path Given a sample.yml file of: ```yaml a: b: cat ``` then ```bash yq '.a.b | path' sample.yml ``` will output ```yaml - a - b ``` ## Get map key Given a sample.yml file of: ```yaml a: b: cat ``` then ```bash yq '.a.b | path | .[-1]' sample.yml ``` will output ```yaml b ``` ## Array path Given a sample.yml file of: ```yaml a: - cat - dog ``` then ```bash yq '.a.[] | select(. == "dog") | path' sample.yml ``` will output ```yaml - a - 1 ``` ## Get array index Given a sample.yml file of: ```yaml a: - cat - dog ``` then ```bash yq '.a.[] | select(. == "dog") | path | .[-1]' sample.yml ``` will output ```yaml 1 ``` ## Print path and value Given a sample.yml file of: ```yaml a: - cat - dog - frog ``` then ```bash yq '.a[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml ``` will output ```yaml - path: - a - 1 value: dog - path: - a - 2 value: frog ``` ## Set path Given a sample.yml file of: ```yaml a: b: cat ``` then ```bash yq 'setpath(["a", "b"]; "things")' sample.yml ``` will output ```yaml a: b: things ``` ## Set on empty document Running ```bash yq --null-input 'setpath(["a", "b"]; "things")' ``` will output ```yaml a: b: things ``` ## Set path to prune deep paths Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object. Given a sample.yml file of: ```yaml parentA: bob parentB: child1: i am child1 child2: i am child2 parentC: child1: me child1 child2: me child2 ``` then ```bash yq '(.parentB.child2, .parentC.child1) as $i ireduce({}; setpath($i | path; $i))' sample.yml ``` will output ```yaml parentB: child2: i am child2 parentC: child1: me child1 ``` ## Set array path Given a sample.yml file of: ```yaml a: - cat - frog ``` then ```bash yq 'setpath(["a", 0]; "things")' sample.yml ``` will output ```yaml a: - things - frog ``` ## Set array path empty Running ```bash yq --null-input 'setpath(["a", 0]; "things")' ``` will output ```yaml a: - things ``` ## Delete path Notice delpaths takes an _array_ of paths. Given a sample.yml file of: ```yaml a: b: cat c: dog d: frog ``` then ```bash yq 'delpaths([["a", "c"], ["a", "d"]])' sample.yml ``` will output ```yaml a: b: cat ``` ## Delete array path Given a sample.yml file of: ```yaml a: - cat - frog ``` then ```bash yq 'delpaths([["a", 0]])' sample.yml ``` will output ```yaml a: - frog ``` ## Delete - wrong parameter delpaths does not work with a single path array Given a sample.yml file of: ```yaml a: - cat - frog ``` then ```bash yq 'delpaths(["a", 0])' sample.yml ``` will output ```bash Error: DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [["a", "b"]] ``` ================================================ FILE: pkg/yqlib/doc/operators/pick.md ================================================ # Pick Filter a map by the specified list of keys. Map is returned with the key in the order of the pick list. Similarly, filter an array by the specified list of indices. ## Pick keys from map Note that the order of the keys matches the pick order and non existent keys are skipped. Given a sample.yml file of: ```yaml myMap: cat: meow dog: bark thing: hamster hamster: squeak ``` then ```bash yq '.myMap |= pick(["hamster", "cat", "goat"])' sample.yml ``` will output ```yaml myMap: hamster: squeak cat: meow ``` ## Pick keys from map, included all the keys We create a map of the picked keys plus all the current keys, and run that through unique Given a sample.yml file of: ```yaml myMap: cat: meow dog: bark thing: hamster hamster: squeak ``` then ```bash yq '.myMap |= pick( (["thing"] + keys) | unique)' sample.yml ``` will output ```yaml myMap: thing: hamster cat: meow dog: bark hamster: squeak ``` ## Pick indices from array Note that the order of the indices matches the pick order and non existent indices are skipped. Given a sample.yml file of: ```yaml - cat - leopard - lion ``` then ```bash yq 'pick([2, 0, 734, -5])' sample.yml ``` will output ```yaml - lion - cat ``` ================================================ FILE: pkg/yqlib/doc/operators/pipe.md ================================================ # Pipe Pipe the results of an expression into another. Like the bash operator. ## Simple Pipe Given a sample.yml file of: ```yaml a: b: cat ``` then ```bash yq '.a | .b' sample.yml ``` will output ```yaml cat ``` ## Multiple updates Given a sample.yml file of: ```yaml a: cow b: sheep c: same ``` then ```bash yq '.a = "cat" | .b = "dog"' sample.yml ``` will output ```yaml a: cat b: dog c: same ``` ================================================ FILE: pkg/yqlib/doc/operators/pivot.md ================================================ # Pivot Emulates the `PIVOT` function supported by several popular RDBMS systems. ## Pivot a sequence of sequences Given a sample.yml file of: ```yaml - - foo - bar - baz - - sis - boom - bah ``` then ```bash yq 'pivot' sample.yml ``` will output ```yaml - - foo - sis - - bar - boom - - baz - bah ``` ## Pivot sequence of heterogeneous sequences Missing values are "padded" to null. Given a sample.yml file of: ```yaml - - foo - bar - baz - - sis - boom - bah - blah ``` then ```bash yq 'pivot' sample.yml ``` will output ```yaml - - foo - sis - - bar - boom - - baz - bah - - - blah ``` ## Pivot sequence of maps Given a sample.yml file of: ```yaml - foo: a bar: b baz: c - foo: x bar: y baz: z ``` then ```bash yq 'pivot' sample.yml ``` will output ```yaml foo: - a - x bar: - b - y baz: - c - z ``` ## Pivot sequence of heterogeneous maps Missing values are "padded" to null. Given a sample.yml file of: ```yaml - foo: a bar: b baz: c - foo: x bar: y baz: z what: ever ``` then ```bash yq 'pivot' sample.yml ``` will output ```yaml foo: - a - x bar: - b - y baz: - c - z what: - - ever ``` ================================================ FILE: pkg/yqlib/doc/operators/recursive-descent-glob.md ================================================ # Recursive Descent (Glob) This operator recursively matches (or globs) all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. ## match values form `..` This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values. For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys: ```bash yq '.. style= "flow"' file.yaml ``` ## match values and map keys form `...` The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling and tags and also use anchors and aliases. For instance to set the `style` of all nodes in a yaml doc, including the map keys: ```bash yq '... style= "flow"' file.yaml ``` ## Recurse map (values only) Given a sample.yml file of: ```yaml a: frog ``` then ```bash yq '..' sample.yml ``` will output ```yaml a: frog frog ``` ## Recursively find nodes with keys Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression. Given a sample.yml file of: ```yaml a: name: frog b: name: blog age: 12 ``` then ```bash yq '[.. | select(has("name"))]' sample.yml ``` will output ```yaml - name: frog b: name: blog age: 12 - name: blog age: 12 ``` ## Recursively find nodes with values Given a sample.yml file of: ```yaml a: nameA: frog b: nameB: frog age: 12 ``` then ```bash yq '.. | select(. == "frog")' sample.yml ``` will output ```yaml frog frog ``` ## Recurse map (values and keys) Note that the map key appears in the results Given a sample.yml file of: ```yaml a: frog ``` then ```bash yq '...' sample.yml ``` will output ```yaml a: frog a frog ``` ## Aliases are not traversed Given a sample.yml file of: ```yaml a: &cat c: frog b: *cat ``` then ```bash yq '[..]' sample.yml ``` will output ```yaml - a: &cat c: frog b: *cat - &cat c: frog - frog - *cat ``` ## Merge docs are not traversed Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar | [..]' sample.yml ``` will output ```yaml - c: foobar_c !!merge <<: *foo thing: foobar_thing - foobar_c - *foo - foobar_thing ``` ================================================ FILE: pkg/yqlib/doc/operators/reduce.md ================================================ # Reduce Reduce is a powerful way to process a collection of data into a new form. ``` as $ ireduce (; ) ``` e.g. ``` .[] as $item ireduce (0; . + $item) ``` On the LHS we are configuring the collection of items that will be reduced `` as well as what each element will be called `$`. Note that the array has been splatted into its individual elements. On the RHS there is ``, the starting value of the accumulator and ``, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. ## yq vs jq syntax Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`). To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added. ## Sum numbers Given a sample.yml file of: ```yaml - 10 - 2 - 5 - 3 ``` then ```bash yq '.[] as $item ireduce (0; . + $item)' sample.yml ``` will output ```yaml 20 ``` ## Merge all yaml files together Given a sample.yml file of: ```yaml a: cat ``` And another sample another.yml file of: ```yaml b: dog ``` then ```bash yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml ``` will output ```yaml a: cat b: dog ``` ## Convert an array to an object Given a sample.yml file of: ```yaml - name: Cathy has: apples - name: Bob has: bananas ``` then ```bash yq '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml ``` will output ```yaml Cathy: apples Bob: bananas ``` ================================================ FILE: pkg/yqlib/doc/operators/reverse.md ================================================ # Reverse Reverses the order of the items in an array ## Reverse Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq 'reverse' sample.yml ``` will output ```yaml - 3 - 2 - 1 ``` ## Sort descending by string field Use sort with reverse to sort in descending order. Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'sort_by(.a) | reverse' sample.yml ``` will output ```yaml - a: cat - a: banana - a: apple ``` ================================================ FILE: pkg/yqlib/doc/operators/select.md ================================================ # Select Select is used to filter arrays and maps by a boolean expression. ## Related Operators - equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals) - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare) - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators) ## Select elements from array using wildcard prefix Given a sample.yml file of: ```yaml - cat - goat - dog ``` then ```bash yq '.[] | select(. == "*at")' sample.yml ``` will output ```yaml cat goat ``` ## Select elements from array using wildcard suffix Given a sample.yml file of: ```yaml - go-kart - goat - dog ``` then ```bash yq '.[] | select(. == "go*")' sample.yml ``` will output ```yaml go-kart goat ``` ## Select elements from array using wildcard prefix and suffix Given a sample.yml file of: ```yaml - ago - go - meow - going ``` then ```bash yq '.[] | select(. == "*go*")' sample.yml ``` will output ```yaml ago go going ``` ## Select elements from array with regular expression See more regular expression examples under the [`string` operator docs](https://mikefarah.gitbook.io/yq/operators/string-operators). Given a sample.yml file of: ```yaml - this_0 - not_this - nor_0_this - thisTo_4 ``` then ```bash yq '.[] | select(test("[a-zA-Z]+_[0-9]$"))' sample.yml ``` will output ```yaml this_0 thisTo_4 ``` ## Select items from a map Given a sample.yml file of: ```yaml things: cat bob: goat horse: dog ``` then ```bash yq '.[] | select(. == "cat" or test("og$"))' sample.yml ``` will output ```yaml cat dog ``` ## Use select and with_entries to filter map keys Given a sample.yml file of: ```yaml name: bob legs: 2 game: poker ``` then ```bash yq 'with_entries(select(.key | test("ame$")))' sample.yml ``` will output ```yaml name: bob game: poker ``` ## Select multiple items in a map and update Note the brackets around the entire LHS. Given a sample.yml file of: ```yaml a: things: cat bob: goat horse: dog ``` then ```bash yq '(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"' sample.yml ``` will output ```yaml a: things: rabbit bob: rabbit horse: dog ``` ================================================ FILE: pkg/yqlib/doc/operators/shuffle.md ================================================ # Shuffle Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order. ## Shuffle array Given a sample.yml file of: ```yaml - 1 - 2 - 3 - 4 - 5 ``` then ```bash yq 'shuffle' sample.yml ``` will output ```yaml - 5 - 2 - 4 - 1 - 3 ``` ## Shuffle array in place Given a sample.yml file of: ```yaml cool: - 1 - 2 - 3 - 4 - 5 ``` then ```bash yq '.cool |= shuffle' sample.yml ``` will output ```yaml cool: - 5 - 2 - 4 - 1 - 3 ``` ================================================ FILE: pkg/yqlib/doc/operators/slice-array.md ================================================ # Slice/Splice Array The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array. You may leave out the first or second number, which will refer to the start or end of the array respectively. ## Slicing arrays Given a sample.yml file of: ```yaml - cat - dog - frog - cow ``` then ```bash yq '.[1:3]' sample.yml ``` will output ```yaml - dog - frog ``` ## Slicing arrays - without the first number Starts from the start of the array Given a sample.yml file of: ```yaml - cat - dog - frog - cow ``` then ```bash yq '.[:2]' sample.yml ``` will output ```yaml - cat - dog ``` ## Slicing arrays - without the second number Finishes at the end of the array Given a sample.yml file of: ```yaml - cat - dog - frog - cow ``` then ```bash yq '.[2:]' sample.yml ``` will output ```yaml - frog - cow ``` ## Slicing arrays - use negative numbers to count backwards from the end Given a sample.yml file of: ```yaml - cat - dog - frog - cow ``` then ```bash yq '.[1:-1]' sample.yml ``` will output ```yaml - dog - frog ``` ## Inserting into the middle of an array using an expression to find the index Given a sample.yml file of: ```yaml - cat - dog - frog - cow ``` then ```bash yq '(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]' sample.yml ``` will output ```yaml - cat - dog - rabbit - frog - cow ``` ================================================ FILE: pkg/yqlib/doc/operators/sort-keys.md ================================================ # Sort Keys The Sort Keys operator sorts maps by their keys (based on their string value). This operator does not do anything to arrays or scalars (so you can easily recursively apply it to all maps). Sort is particularly useful for diffing two different yaml documents: ```bash yq -i -P 'sort_keys(..)' file1.yml yq -i -P 'sort_keys(..)' file2.yml diff file1.yml file2.yml ``` Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if you are using merge anchors. For more advanced sorting, you can use the [sort_by](https://mikefarah.gitbook.io/yq/operators/sort) function on a map, and give it a custom function like `sort_by(key | downcase)`. ## Sort keys of map Given a sample.yml file of: ```yaml c: frog a: blah b: bing ``` then ```bash yq 'sort_keys(.)' sample.yml ``` will output ```yaml a: blah b: bing c: frog ``` ## Sort keys recursively Note the array elements are left unsorted, but maps inside arrays are sorted Given a sample.yml file of: ```yaml bParent: c: dog array: - 3 - 1 - 2 aParent: z: donkey x: - c: yum b: delish - b: ew a: apple ``` then ```bash yq 'sort_keys(..)' sample.yml ``` will output ```yaml aParent: x: - b: delish c: yum - a: apple b: ew z: donkey bParent: array: - 3 - 1 - 2 c: dog ``` ================================================ FILE: pkg/yqlib/doc/operators/sort.md ================================================ # Sort Sorts an array. Use `sort` to sort an array as is, or `sort_by(exp)` to sort by a particular expression (e.g. subfield). To sort by descending order, pipe the results through the `reverse` operator after sorting. Note that at this stage, `yq` only sorts scalar fields. ## Sort by string field Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'sort_by(.a)' sample.yml ``` will output ```yaml - a: apple - a: banana - a: cat ``` ## Sort by multiple fields Given a sample.yml file of: ```yaml - a: dog - a: cat b: banana - a: cat b: apple ``` then ```bash yq 'sort_by(.a, .b)' sample.yml ``` will output ```yaml - a: cat b: apple - a: cat b: banana - a: dog ``` ## Sort descending by string field Use sort with reverse to sort in descending order. Given a sample.yml file of: ```yaml - a: banana - a: cat - a: apple ``` then ```bash yq 'sort_by(.a) | reverse' sample.yml ``` will output ```yaml - a: cat - a: banana - a: apple ``` ## Sort array in place Given a sample.yml file of: ```yaml cool: - a: banana - a: cat - a: apple ``` then ```bash yq '.cool |= sort_by(.a)' sample.yml ``` will output ```yaml cool: - a: apple - a: banana - a: cat ``` ## Sort array of objects by key Note that you can give sort_by complex expressions, not just paths Given a sample.yml file of: ```yaml cool: - b: banana - a: banana - c: banana ``` then ```bash yq '.cool |= sort_by(keys | .[0])' sample.yml ``` will output ```yaml cool: - a: banana - b: banana - c: banana ``` ## Sort a map Sorting a map, by default this will sort by the values Given a sample.yml file of: ```yaml y: b z: a x: c ``` then ```bash yq 'sort' sample.yml ``` will output ```yaml z: a y: b x: c ``` ## Sort a map by keys Use sort_by to sort a map using a custom function Given a sample.yml file of: ```yaml Y: b z: a x: c ``` then ```bash yq 'sort_by(key | downcase)' sample.yml ``` will output ```yaml x: c Y: b z: a ``` ## Sort is stable Note the order of the elements in unchanged when equal in sorting. Given a sample.yml file of: ```yaml - a: banana b: 1 - a: banana b: 2 - a: banana b: 3 - a: banana b: 4 ``` then ```bash yq 'sort_by(.a)' sample.yml ``` will output ```yaml - a: banana b: 1 - a: banana b: 2 - a: banana b: 3 - a: banana b: 4 ``` ## Sort by numeric field Given a sample.yml file of: ```yaml - a: 10 - a: 100 - a: 1 ``` then ```bash yq 'sort_by(.a)' sample.yml ``` will output ```yaml - a: 1 - a: 10 - a: 100 ``` ## Sort by custom date field Given a sample.yml file of: ```yaml - a: 12-Jun-2011 - a: 23-Dec-2010 - a: 10-Aug-2011 ``` then ```bash yq 'with_dtf("02-Jan-2006"; sort_by(.a))' sample.yml ``` will output ```yaml - a: 23-Dec-2010 - a: 12-Jun-2011 - a: 10-Aug-2011 ``` ## Sort, nulls come first Given a sample.yml file of: ```yaml - 8 - 3 - null - 6 - true - false - cat ``` then ```bash yq 'sort' sample.yml ``` will output ```yaml - null - false - true - 3 - 6 - 8 - cat ``` ================================================ FILE: pkg/yqlib/doc/operators/split-into-documents.md ================================================ # Split into Documents This operator splits all matches into separate documents ## Split empty Running ```bash yq --null-input 'split_doc' ``` will output ```yaml ``` ## Split array Given a sample.yml file of: ```yaml - a: cat - b: dog ``` then ```bash yq '.[] | split_doc' sample.yml ``` will output ```yaml a: cat --- b: dog ``` ================================================ FILE: pkg/yqlib/doc/operators/string-operators.md ================================================ # String Operators ## RegEx This uses Golang's native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax. Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats")`. ### match(regEx) This operator returns the substring match details of the given regEx. ### capture(regEx) Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing. ## test(regEx) Returns true if the string matches the RegEx, false otherwise. ## sub(regEx, replacement) Substitutes matched substrings. The first parameter is the regEx to match substrings within the original string. The second parameter specifies what to replace those matches with. This can refer to capture groups from the first RegEx. ## String blocks, bash and newlines Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_. For instance to get this yaml: ``` a: | cat ``` Using `$( exp )` wont work, as it will trim the trailing newline. ``` m=$(echo "cat\n") yq -n '.a = strenv(m)' a: cat ``` However, using printf works: ``` printf -v m "cat\n" ; m="$m" yq -n '.a = strenv(m)' a: | cat ``` As well as having multiline expressions: ``` m="cat " yq -n '.a = strenv(m)' a: | cat ``` Similarly, if you're trying to set the content from a file, and want a trailing newline: ``` IFS= read -rd '' output < <(cat my_file) output=$output ./yq '.data.values = strenv(output)' first.yml ``` ## Interpolation Given a sample.yml file of: ```yaml value: things another: stuff ``` then ```bash yq '.message = "I like \(.value) and \(.another)"' sample.yml ``` will output ```yaml value: things another: stuff message: I like things and stuff ``` ## Interpolation - not a string Given a sample.yml file of: ```yaml value: an: apple ``` then ```bash yq '.message = "I like \(.value)"' sample.yml ``` will output ```yaml value: an: apple message: 'I like an: apple' ``` ## To up (upper) case Works with unicode characters Given a sample.yml file of: ```yaml água ``` then ```bash yq 'upcase' sample.yml ``` will output ```yaml ÁGUA ``` ## To down (lower) case Works with unicode characters Given a sample.yml file of: ```yaml ÁgUA ``` then ```bash yq 'downcase' sample.yml ``` will output ```yaml água ``` ## Join strings Given a sample.yml file of: ```yaml - cat - meow - 1 - null - true ``` then ```bash yq 'join("; ")' sample.yml ``` will output ```yaml cat; meow; 1; ; true ``` ## Trim strings Given a sample.yml file of: ```yaml - ' cat' - 'dog ' - ' cow cow ' - horse ``` then ```bash yq '.[] | trim' sample.yml ``` will output ```yaml cat dog cow cow horse ``` ## Match string Given a sample.yml file of: ```yaml foo bar foo ``` then ```bash yq 'match("foo")' sample.yml ``` will output ```yaml string: foo offset: 0 length: 3 captures: [] ``` ## Match string, case insensitive Given a sample.yml file of: ```yaml foo bar FOO ``` then ```bash yq '[match("(?i)foo"; "g")]' sample.yml ``` will output ```yaml - string: foo offset: 0 length: 3 captures: [] - string: FOO offset: 8 length: 3 captures: [] ``` ## Match with global capture group Given a sample.yml file of: ```yaml abc abc ``` then ```bash yq '[match("(ab)(c)"; "g")]' sample.yml ``` will output ```yaml - string: abc offset: 0 length: 3 captures: - string: ab offset: 0 length: 2 - string: c offset: 2 length: 1 - string: abc offset: 4 length: 3 captures: - string: ab offset: 4 length: 2 - string: c offset: 6 length: 1 ``` ## Match with named capture groups Given a sample.yml file of: ```yaml foo bar foo foo foo ``` then ```bash yq '[match("foo (?Pbar)? foo"; "g")]' sample.yml ``` will output ```yaml - string: foo bar foo offset: 0 length: 11 captures: - string: bar offset: 4 length: 3 name: bar123 - string: foo foo offset: 12 length: 8 captures: - string: null offset: -1 length: 0 name: bar123 ``` ## Capture named groups into a map Given a sample.yml file of: ```yaml xyzzy-14 ``` then ```bash yq 'capture("(?P[a-z]+)-(?P[0-9]+)")' sample.yml ``` will output ```yaml a: xyzzy n: "14" ``` ## Match without global flag Given a sample.yml file of: ```yaml cat cat ``` then ```bash yq 'match("cat")' sample.yml ``` will output ```yaml string: cat offset: 0 length: 3 captures: [] ``` ## Match with global flag Given a sample.yml file of: ```yaml cat cat ``` then ```bash yq '[match("cat"; "g")]' sample.yml ``` will output ```yaml - string: cat offset: 0 length: 3 captures: [] - string: cat offset: 4 length: 3 captures: [] ``` ## Test using regex Like jq's equivalent, this works like match but only returns true/false instead of full match details Given a sample.yml file of: ```yaml - cat - dog ``` then ```bash yq '.[] | test("at")' sample.yml ``` will output ```yaml true false ``` ## Substitute / Replace string This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax). Note the use of `|=` to run in context of the current string value. Given a sample.yml file of: ```yaml a: dogs are great ``` then ```bash yq '.a |= sub("dogs", "cats")' sample.yml ``` will output ```yaml a: cats are great ``` ## Substitute / Replace string with regex This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax). Note the use of `|=` to run in context of the current string value. Given a sample.yml file of: ```yaml a: cat b: heat ``` then ```bash yq '.[] |= sub("(a)", "${1}r")' sample.yml ``` will output ```yaml a: cart b: heart ``` ## Custom types: that are really strings When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml a: !horse cat b: !goat heat ``` then ```bash yq '.[] |= sub("(a)", "${1}r")' sample.yml ``` will output ```yaml a: !horse cart b: !goat heart ``` ## Split strings Given a sample.yml file of: ```yaml cat; meow; 1; ; true ``` then ```bash yq 'split("; ")' sample.yml ``` will output ```yaml - cat - meow - "1" - "" - "true" ``` ## Split strings one match Given a sample.yml file of: ```yaml word ``` then ```bash yq 'split("; ")' sample.yml ``` will output ```yaml - word ``` ## To string Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f` Given a sample.yml file of: ```yaml - 1 - true - null - ~ - cat - an: object - - array - 2 ``` then ```bash yq '.[] |= to_string' sample.yml ``` will output ```yaml - "1" - "true" - "null" - "~" - cat - "an: object" - "- array\n- 2" ``` ================================================ FILE: pkg/yqlib/doc/operators/style.md ================================================ # Style The style operator can be used to get or set the style of nodes (e.g. string style, yaml style). Use this to control the formatting of the document in yaml. ## Update and set style of a particular node (simple) Given a sample.yml file of: ```yaml a: b: thing c: something ``` then ```bash yq '.a.b = "new" | .a.b style="double"' sample.yml ``` will output ```yaml a: b: "new" c: something ``` ## Update and set style of a particular node using path variables Given a sample.yml file of: ```yaml a: b: thing c: something ``` then ```bash yq 'with(.a.b ; . = "new" | . style="double")' sample.yml ``` will output ```yaml a: b: "new" c: something ``` ## Set tagged style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="tagged"' sample.yml ``` will output ```yaml !!map a: !!str cat b: !!int 5 c: !!float 3.2 e: !!bool true f: !!seq - !!int 1 - !!int 2 - !!int 3 g: !!map something: !!str cool ``` ## Set double quote style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="double"' sample.yml ``` will output ```yaml a: "cat" b: "5" c: "3.2" e: "true" f: - "1" - "2" - "3" g: something: "cool" ``` ## Set double quote style on map keys too Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '... style="double"' sample.yml ``` will output ```yaml "a": "cat" "b": "5" "c": "3.2" "e": "true" "f": - "1" - "2" - "3" "g": "something": "cool" ``` ## Set single quote style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="single"' sample.yml ``` will output ```yaml a: 'cat' b: '5' c: '3.2' e: 'true' f: - '1' - '2' - '3' g: something: 'cool' ``` ## Set literal quote style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="literal"' sample.yml ``` will output ```yaml a: |- cat b: |- 5 c: |- 3.2 e: |- true f: - |- 1 - |- 2 - |- 3 g: something: |- cool ``` ## Set folded quote style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="folded"' sample.yml ``` will output ```yaml a: >- cat b: >- 5 c: >- 3.2 e: >- true f: - >- 1 - >- 2 - >- 3 g: something: >- cool ``` ## Set flow quote style Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` then ```bash yq '.. style="flow"' sample.yml ``` will output ```yaml {a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}} ``` ## Reset style - or pretty print Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this. Given a sample.yml file of: ```yaml {a: cat, "b": 5, 'c': 3.2, "e": true, f: [1,2,3], "g": { something: "cool"} } ``` then ```bash yq '... style=""' sample.yml ``` will output ```yaml a: cat b: 5 c: 3.2 e: true f: - 1 - 2 - 3 g: something: cool ``` ## Set style relatively with assign-update Given a sample.yml file of: ```yaml a: single b: double ``` then ```bash yq '.[] style |= .' sample.yml ``` will output ```yaml a: 'single' b: "double" ``` ## Read style Given a sample.yml file of: ```yaml {a: "cat", b: 'thing'} ``` then ```bash yq '.. | style' sample.yml ``` will output ```yaml flow double single ``` ================================================ FILE: pkg/yqlib/doc/operators/subtract.md ================================================ # Subtract You can use subtract to subtract numbers as well as remove elements from an array. ## Array subtraction Running ```bash yq --null-input '[1,2] - [2,3]' ``` will output ```yaml - 1 ``` ## Array subtraction with nested array Running ```bash yq --null-input '[[1], 1, 2] - [[1], 3]' ``` will output ```yaml - 1 - 2 ``` ## Array subtraction with nested object Note that order of the keys does not matter Given a sample.yml file of: ```yaml - a: b c: d - a: b ``` then ```bash yq '. - [{"c": "d", "a": "b"}]' sample.yml ``` will output ```yaml - a: b ``` ## Number subtraction - float If the lhs or rhs are floats then the expression will be calculated with floats. Given a sample.yml file of: ```yaml a: 3 b: 4.5 ``` then ```bash yq '.a = .a - .b' sample.yml ``` will output ```yaml a: -1.5 b: 4.5 ``` ## Number subtraction - int If both the lhs and rhs are ints then the expression will be calculated with ints. Given a sample.yml file of: ```yaml a: 3 b: 4 ``` then ```bash yq '.a = .a - .b' sample.yml ``` will output ```yaml a: -1 b: 4 ``` ## Decrement numbers Given a sample.yml file of: ```yaml a: 3 b: 5 ``` then ```bash yq '.[] -= 1' sample.yml ``` will output ```yaml a: 2 b: 4 ``` ## Date subtraction You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. Given a sample.yml file of: ```yaml a: 2021-01-01T03:10:00Z ``` then ```bash yq '.a -= "3h10m"' sample.yml ``` will output ```yaml a: 2021-01-01T00:00:00Z ``` ## Date subtraction - custom format Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. Given a sample.yml file of: ```yaml a: Saturday, 15-Dec-01 at 6:00AM GMT ``` then ```bash yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")' sample.yml ``` will output ```yaml a: Saturday, 15-Dec-01 at 2:59AM GMT ``` ## Custom types: that are really numbers When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml a: !horse 2 b: !goat 1 ``` then ```bash yq '.a -= .b' sample.yml ``` will output ```yaml a: !horse 1 b: !goat 1 ``` ================================================ FILE: pkg/yqlib/doc/operators/tag.md ================================================ # Tag The tag operator can be used to get or set the tag of nodes (e.g. `!!str`, `!!int`, `!!bool`). ## Get tag Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: [] ``` then ```bash yq '.. | tag' sample.yml ``` will output ```yaml !!map !!str !!int !!float !!bool !!seq ``` ## type is an alias for tag Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true f: [] ``` then ```bash yq '.. | type' sample.yml ``` will output ```yaml !!map !!str !!int !!float !!bool !!seq ``` ## Set custom tag Given a sample.yml file of: ```yaml a: str ``` then ```bash yq '.a tag = "!!mikefarah"' sample.yml ``` will output ```yaml a: !!mikefarah str ``` ## Find numbers and convert them to strings Given a sample.yml file of: ```yaml a: cat b: 5 c: 3.2 e: true ``` then ```bash yq '(.. | select(tag == "!!int")) tag= "!!str"' sample.yml ``` will output ```yaml a: cat b: "5" c: 3.2 e: true ``` ================================================ FILE: pkg/yqlib/doc/operators/to_number.md ================================================ # To Number Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. ## Converts strings to numbers Given a sample.yml file of: ```yaml - "3" - "3.1" - "-1e3" ``` then ```bash yq '.[] | to_number' sample.yml ``` will output ```yaml 3 3.1 -1e3 ``` ## Doesn't change numbers Given a sample.yml file of: ```yaml - 3 - 3.1 - -1e3 ``` then ```bash yq '.[] | to_number' sample.yml ``` will output ```yaml 3 3.1 -1e3 ``` ## Cannot convert null Running ```bash yq --null-input '.a.b | to_number' ``` will output ```bash Error: cannot convert node value [null] at path a.b of tag !!null to number ``` ================================================ FILE: pkg/yqlib/doc/operators/traverse-read.md ================================================ # Traverse (Read) This is the simplest (and perhaps most used) operator. It is used to navigate deeply into yaml structures. ## NOTE --yaml-fix-merge-anchor-to-spec flag `yq` doesn't merge anchors `<<:` to spec, in some circumstances it incorrectly overrides existing keys when the spec documents not to do that. To minimise disruption while still fixing the issue, a flag has been added to toggle this behaviour. This will first default to false; and log warnings to users. Then it will default to true (and still allow users to specify false if needed) See examples of the flag differences below, where LEGACY is the flag off; and FIXED is with the flag on. ## Simple map navigation Given a sample.yml file of: ```yaml a: b: apple ``` then ```bash yq '.a' sample.yml ``` will output ```yaml b: apple ``` ## Splat Often used to pipe children into other operators Given a sample.yml file of: ```yaml - b: apple - c: banana ``` then ```bash yq '.[]' sample.yml ``` will output ```yaml b: apple c: banana ``` ## Optional Splat Just like splat, but won't error if you run it against scalars Given a sample.yml file of: ```yaml cat ``` then ```bash yq '.[]' sample.yml ``` will output ```yaml ``` ## Special characters Use quotes with square brackets around path elements with special characters Given a sample.yml file of: ```yaml "{}": frog ``` then ```bash yq '.["{}"]' sample.yml ``` will output ```yaml frog ``` ## Nested special characters Given a sample.yml file of: ```yaml a: "key.withdots": "another.key": apple ``` then ```bash yq '.a["key.withdots"]["another.key"]' sample.yml ``` will output ```yaml apple ``` ## Keys with spaces Use quotes with square brackets around path elements with special characters Given a sample.yml file of: ```yaml "red rabbit": frog ``` then ```bash yq '.["red rabbit"]' sample.yml ``` will output ```yaml frog ``` ## Dynamic keys Expressions within [] can be used to dynamically lookup / calculate keys Given a sample.yml file of: ```yaml b: apple apple: crispy yum banana: soft yum ``` then ```bash yq '.[.b]' sample.yml ``` will output ```yaml crispy yum ``` ## Children don't exist Nodes are added dynamically while traversing Given a sample.yml file of: ```yaml c: banana ``` then ```bash yq '.a.b' sample.yml ``` will output ```yaml null ``` ## Optional identifier Like jq, does not output an error when the yaml is not an array or object as expected Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq '.a?' sample.yml ``` will output ```yaml ``` ## Wildcard matching Given a sample.yml file of: ```yaml a: cat: apple mad: things ``` then ```bash yq '.a."*a*"' sample.yml ``` will output ```yaml apple things ``` ## Aliases Given a sample.yml file of: ```yaml a: &cat c: frog b: *cat ``` then ```bash yq '.b' sample.yml ``` will output ```yaml *cat ``` ## Traversing aliases with splat Given a sample.yml file of: ```yaml a: &cat c: frog b: *cat ``` then ```bash yq '.b[]' sample.yml ``` will output ```yaml frog ``` ## Traversing aliases explicitly Given a sample.yml file of: ```yaml a: &cat c: frog b: *cat ``` then ```bash yq '.b.c' sample.yml ``` will output ```yaml frog ``` ## Traversing arrays by index Given a sample.yml file of: ```yaml - 1 - 2 - 3 ``` then ```bash yq '.[0]' sample.yml ``` will output ```yaml 1 ``` ## Traversing nested arrays by index Given a sample.yml file of: ```yaml [[], [cat]] ``` then ```bash yq '.[1][0]' sample.yml ``` will output ```yaml cat ``` ## Maps with numeric keys Given a sample.yml file of: ```yaml 2: cat ``` then ```bash yq '.[2]' sample.yml ``` will output ```yaml cat ``` ## Maps with non existing numeric keys Given a sample.yml file of: ```yaml a: b ``` then ```bash yq '.[0]' sample.yml ``` will output ```yaml null ``` ## Traversing merge anchors Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar.a' sample.yml ``` will output ```yaml foo_a ``` ## Traversing merge anchors with local override Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar.thing' sample.yml ``` will output ```yaml foobar_thing ``` ## Select multiple indices Given a sample.yml file of: ```yaml a: - a - b - c ``` then ```bash yq '.a[0, 2]' sample.yml ``` will output ```yaml a c ``` ## LEGACY: Traversing merge anchors with override This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar.c' sample.yml ``` will output ```yaml foo_c ``` ## LEGACY: Traversing merge anchor lists Note that the later merge anchors override previous, but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobarList.thing' sample.yml ``` will output ```yaml bar_thing ``` ## LEGACY: Splatting merge anchors With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar[]' sample.yml ``` will output ```yaml foo_c foo_a foobar_thing ``` ## LEGACY: Splatting merge anchor lists With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobarList[]' sample.yml ``` will output ```yaml bar_b foo_a bar_thing foobarList_c ``` ## FIXED: Traversing merge anchors with override Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar.c' sample.yml ``` will output ```yaml foobar_c ``` ## FIXED: Traversing merge anchor lists Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobarList.thing' sample.yml ``` will output ```yaml foo_thing ``` ## FIXED: Splatting merge anchors Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobar[]' sample.yml ``` will output ```yaml foo_a foobar_thing foobar_c ``` ## FIXED: Splatting merge anchor lists Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones Given a sample.yml file of: ```yaml foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b !!merge <<: - *foo - *bar c: foobarList_c foobar: c: foobar_c !!merge <<: *foo thing: foobar_thing ``` then ```bash yq '.foobarList[]' sample.yml ``` will output ```yaml foobarList_b foo_thing foobarList_c foo_a ``` ================================================ FILE: pkg/yqlib/doc/operators/union.md ================================================ # Union This operator is used to combine different results together. ## Combine scalars Running ```bash yq --null-input '1, true, "cat"' ``` will output ```yaml 1 true cat ``` ## Combine selected paths Given a sample.yml file of: ```yaml a: fieldA b: fieldB c: fieldC ``` then ```bash yq '.a, .c' sample.yml ``` will output ```yaml fieldA fieldC ``` ================================================ FILE: pkg/yqlib/doc/operators/unique.md ================================================ # Unique This is used to filter out duplicated items in an array. Note that the original order of the array is maintained. ## Unique array of scalars (string/numbers) Note that unique maintains the original order of the array. Given a sample.yml file of: ```yaml - 2 - 1 - 3 - 2 ``` then ```bash yq 'unique' sample.yml ``` will output ```yaml - 2 - 1 - 3 ``` ## Unique nulls Unique works on the node value, so it considers different representations of nulls to be different Given a sample.yml file of: ```yaml - ~ - null - ~ - null ``` then ```bash yq 'unique' sample.yml ``` will output ```yaml - ~ - null ``` ## Unique all nulls Run against the node tag to unique all the nulls Given a sample.yml file of: ```yaml - ~ - null - ~ - null ``` then ```bash yq 'unique_by(tag)' sample.yml ``` will output ```yaml - ~ ``` ## Unique array objects Given a sample.yml file of: ```yaml - name: harry pet: cat - name: billy pet: dog - name: harry pet: cat ``` then ```bash yq 'unique' sample.yml ``` will output ```yaml - name: harry pet: cat - name: billy pet: dog ``` ## Unique array of objects by a field Given a sample.yml file of: ```yaml - name: harry pet: cat - name: billy pet: dog - name: harry pet: dog ``` then ```bash yq 'unique_by(.name)' sample.yml ``` will output ```yaml - name: harry pet: cat - name: billy pet: dog ``` ## Unique array of arrays Given a sample.yml file of: ```yaml - - cat - dog - - cat - sheep - - cat - dog ``` then ```bash yq 'unique' sample.yml ``` will output ```yaml - - cat - dog - - cat - sheep ``` ================================================ FILE: pkg/yqlib/doc/operators/variable-operators.md ================================================ # Variable Operators Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. ## Single value variable Given a sample.yml file of: ```yaml a: cat ``` then ```bash yq '.a as $foo | $foo' sample.yml ``` will output ```yaml cat ``` ## Multi value variable Given a sample.yml file of: ```yaml - cat - dog ``` then ```bash yq '.[] as $foo | $foo' sample.yml ``` will output ```yaml cat dog ``` ## Using variables as a lookup Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...) Given a sample.yml file of: ```yaml "posts": - "title": First post "author": anon - "title": A well-written article "author": person1 "realnames": "anon": Anonymous Coward "person1": Person McPherson ``` then ```bash yq '.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}' sample.yml ``` will output ```yaml title: First post author: Anonymous Coward title: A well-written article author: Person McPherson ``` ## Using variables to swap values Given a sample.yml file of: ```yaml a: a_value b: b_value ``` then ```bash yq '.a as $x | .b as $y | .b = $x | .a = $y' sample.yml ``` will output ```yaml a: b_value b: a_value ``` ## Use ref to reference a path repeatedly Note: You may find the `with` operator more useful. Given a sample.yml file of: ```yaml a: b: thing c: something ``` then ```bash yq '.a.b ref $x | $x = "new" | $x style="double"' sample.yml ``` will output ```yaml a: b: "new" c: something ``` ================================================ FILE: pkg/yqlib/doc/operators/with.md ================================================ # With Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context. ## Update and style Given a sample.yml file of: ```yaml a: deeply: nested: value ``` then ```bash yq 'with(.a.deeply.nested; . = "newValue" | . style="single")' sample.yml ``` will output ```yaml a: deeply: nested: 'newValue' ``` ## Update multiple deeply nested properties Given a sample.yml file of: ```yaml a: deeply: nested: value other: thing ``` then ```bash yq 'with(.a.deeply; .nested = "newValue" | .other= "newThing")' sample.yml ``` will output ```yaml a: deeply: nested: newValue other: newThing ``` ## Update array elements relatively The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element. Given a sample.yml file of: ```yaml myArray: - a: apple - a: banana ``` then ```bash yq 'with(.myArray[]; .b = .a + " yum")' sample.yml ``` will output ```yaml myArray: - a: apple b: apple yum - a: banana b: banana yum ``` ================================================ FILE: pkg/yqlib/doc/usage/base64.md ================================================ # Base64 Encode and decode to and from Base64. Base64 assumes [RFC4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a UTF-8 string and not binary content. See below for examples ## Decode base64: simple Decoded data is assumed to be a string. Given a sample.txt file of: ``` YSBzcGVjaWFsIHN0cmluZw== ``` then ```bash yq -p=base64 -oy '.' sample.txt ``` will output ```yaml a special string ``` ## Decode base64: UTF-8 Base64 decoding supports UTF-8 encoded strings. Given a sample.txt file of: ``` V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig== ``` then ```bash yq -p=base64 -oy '.' sample.txt ``` will output ```yaml Works with UTF-16 😊 ``` ## Decode with extra spaces Extra leading/trailing whitespace is stripped Given a sample.txt file of: ``` YSBzcGVjaWFsIHN0cmluZw== ``` then ```bash yq -p=base64 -oy '.' sample.txt ``` will output ```yaml a special string ``` ## Encode base64: string Given a sample.yml file of: ```yaml "a special string" ``` then ```bash yq -o=base64 '.' sample.yml ``` will output ``` YSBzcGVjaWFsIHN0cmluZw==``` ## Encode base64: string from document Extract a string field and encode it to base64. Given a sample.yml file of: ```yaml coolData: "a special string" ``` then ```bash yq -o=base64 '.coolData' sample.yml ``` will output ``` YSBzcGVjaWFsIHN0cmluZw==``` ================================================ FILE: pkg/yqlib/doc/usage/convert.md ================================================ # JSON Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON). Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. ## Parse json: simple JSON is a subset of yaml, so all you need to do is prettify the output Given a sample.json file of: ```json {"cat": "meow"} ``` then ```bash yq -p=json sample.json ``` will output ```yaml cat: meow ``` ## Parse json: complex JSON is a subset of yaml, so all you need to do is prettify the output Given a sample.json file of: ```json {"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}} ``` then ```bash yq -p=json sample.json ``` will output ```yaml a: Easy! as one two three b: c: 2 d: - 3 - 4 ``` ## Encode json: simple Given a sample.yml file of: ```yaml cat: meow ``` then ```bash yq -o=json '.' sample.yml ``` will output ```json { "cat": "meow" } ``` ## Encode json: simple - in one line Given a sample.yml file of: ```yaml cat: meow # this is a comment, and it will be dropped. ``` then ```bash yq -o=json -I=0 '.' sample.yml ``` will output ```json {"cat":"meow"} ``` ## Encode json: comments Given a sample.yml file of: ```yaml cat: meow # this is a comment, and it will be dropped. ``` then ```bash yq -o=json '.' sample.yml ``` will output ```json { "cat": "meow" } ``` ## Encode json: anchors Anchors are dereferenced Given a sample.yml file of: ```yaml cat: &ref meow anotherCat: *ref ``` then ```bash yq -o=json '.' sample.yml ``` will output ```json { "cat": "meow", "anotherCat": "meow" } ``` ## Encode json: multiple results Each matching node is converted into a json doc. This is best used with 0 indent (json document per line) Given a sample.yml file of: ```yaml things: [{stuff: cool}, {whatever: cat}] ``` then ```bash yq -o=json -I=0 '.things[]' sample.yml ``` will output ```json {"stuff":"cool"} {"whatever":"cat"} ``` ## Roundtrip JSON Lines / NDJSON Given a sample.json file of: ```json {"this": "is a multidoc json file"} {"each": ["line is a valid json document"]} {"a number": 4} ``` then ```bash yq -p=json -o=json -I=0 sample.json ``` will output ```yaml {"this":"is a multidoc json file"} {"each":["line is a valid json document"]} {"a number":4} ``` ## Roundtrip multi-document JSON The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents Given a sample.json file of: ```json { "this": "is a multidoc json file" } { "it": [ "has", "consecutive", "json documents" ] } { "a number": 4 } ``` then ```bash yq -p=json -o=json -I=2 sample.json ``` will output ```yaml { "this": "is a multidoc json file" } { "it": [ "has", "consecutive", "json documents" ] } { "a number": 4 } ``` ## Update a specific document in a multi-document json Documents are indexed by the `documentIndex` or `di` operator. Given a sample.json file of: ```json {"this": "is a multidoc json file"} {"each": ["line is a valid json document"]} {"a number": 4} ``` then ```bash yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json ``` will output ```yaml {"this":"is a multidoc json file"} {"each":["line is a valid json document","cool"]} {"a number":4} ``` ## Find and update a specific document in a multi-document json Use expressions as you normally would. Given a sample.json file of: ```json {"this": "is a multidoc json file"} {"each": ["line is a valid json document"]} {"a number": 4} ``` then ```bash yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json ``` will output ```yaml {"this":"is a multidoc json file"} {"each":["line is a valid json document","cool"]} {"a number":4} ``` ## Decode JSON Lines / NDJSON Given a sample.json file of: ```json {"this": "is a multidoc json file"} {"each": ["line is a valid json document"]} {"a number": 4} ``` then ```bash yq -p=json sample.json ``` will output ```yaml this: is a multidoc json file --- each: - line is a valid json document --- a number: 4 ``` ================================================ FILE: pkg/yqlib/doc/usage/csv-tsv.md ================================================ # CSV Encode/Decode/Roundtrip CSV and TSV files. ## Encode Currently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: ```yaml - name: Bobo type: dog - name: Fifi type: cat ``` As well as arrays of arrays of scalars (strings/numbers/booleans): ```yaml - [Bobo, dog] - [Fifi, cat] ``` ## Decode Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries. The data will be coded into an array of objects, using the header rows as keys. ```csv name,type Bobo,dog Fifi,cat ``` ## Encode CSV simple Given a sample.yml file of: ```yaml - [i, like, csv] - [because, excel, is, cool] ``` then ```bash yq -o=csv sample.yml ``` will output ```csv i,like,csv because,excel,is,cool ``` ## Encode TSV simple Given a sample.yml file of: ```yaml - [i, like, csv] - [because, excel, is, cool] ``` then ```bash yq -o=tsv sample.yml ``` will output ```tsv i like csv because excel is cool ``` ## Encode array of objects to csv Given a sample.yml file of: ```yaml - name: Gary numberOfCats: 1 likesApples: true height: 168.8 - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 ``` then ```bash yq -o=csv sample.yml ``` will output ```csv name,numberOfCats,likesApples,height Gary,1,true,168.8 Samantha's Rabbit,2,false,-188.8 ``` ## Encode array of objects to custom csv format Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like. Given a sample.yml file of: ```yaml - name: Gary numberOfCats: 1 likesApples: true height: 168.8 - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 ``` then ```bash yq -o=csv '[["Name", "Number of Cats"]] + [.[] | [.name, .numberOfCats ]]' sample.yml ``` will output ```csv Name,Number of Cats Gary,1 Samantha's Rabbit,2 ``` ## Encode array of objects to csv - missing fields behaviour First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank Given a sample.yml file of: ```yaml - name: Gary numberOfCats: 1 height: 168.8 - name: Samantha's Rabbit height: -188.8 likesApples: false ``` then ```bash yq -o=csv sample.yml ``` will output ```csv name,numberOfCats,height Gary,1,168.8 Samantha's Rabbit,,-188.8 ``` ## Parse CSV into an array of objects First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed! Given a sample.csv file of: ```csv name,numberOfCats,likesApples,height,facts Gary,1,true,168.8,cool: true Samantha's Rabbit,2,false,-188.8,tall: indeed ``` then ```bash yq -p=csv sample.csv ``` will output ```yaml - name: Gary numberOfCats: 1 likesApples: true height: 168.8 facts: cool: true - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 facts: tall: indeed ``` ## Parse CSV into an array of objects, no auto-parsing First row is assumed to be the header row. Entries with YAML/JSON will be left as strings. Given a sample.csv file of: ```csv name,numberOfCats,likesApples,height,facts Gary,1,true,168.8,cool: true Samantha's Rabbit,2,false,-188.8,tall: indeed ``` then ```bash yq -p=csv --csv-auto-parse=f sample.csv ``` will output ```yaml - name: Gary numberOfCats: 1 likesApples: true height: 168.8 facts: 'cool: true' - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 facts: 'tall: indeed' ``` ## Parse TSV into an array of objects First row is assumed to be the header row. Given a sample.tsv file of: ```tsv name numberOfCats likesApples height Gary 1 true 168.8 Samantha's Rabbit 2 false -188.8 ``` then ```bash yq -p=tsv sample.tsv ``` will output ```yaml - name: Gary numberOfCats: 1 likesApples: true height: 168.8 - name: Samantha's Rabbit numberOfCats: 2 likesApples: false height: -188.8 ``` ## Round trip Given a sample.csv file of: ```csv name,numberOfCats,likesApples,height Gary,1,true,168.8 Samantha's Rabbit,2,false,-188.8 ``` then ```bash yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv ``` will output ```csv name,numberOfCats,likesApples,height Gary,3,true,168.8 Samantha's Rabbit,2,false,-188.8 ``` ================================================ FILE: pkg/yqlib/doc/usage/formatting-expressions.md ================================================ # Formatting Expressions `From version v4.41+` You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on. ## Using expression files and comments Note that you can execute the file directly - but make sure you make the expression file executable. Given a sample.yaml file of: ```yaml a: b: old ``` And an 'update.yq' expression file of: ```bash #! yq # This is a yq expression that updates the map # for several great reasons outlined here. .a.b = "new" # line comment here | .a.c = "frog" # Now good things will happen. ``` then ```bash ./update.yq sample.yaml ``` will output ```yaml a: b: new c: frog ``` ## Flags in expression files You can specify flags on the shebang line, this only works when executing the file directly. Given a sample.yaml file of: ```yaml a: b: old ``` And an 'update.yq' expression file of: ```bash #! yq -oj # This is a yq expression that updates the map # for several great reasons outlined here. .a.b = "new" # line comment here | .a.c = "frog" # Now good things will happen. ``` then ```bash ./update.yq sample.yaml ``` will output ```yaml { "a": { "b": "new", "c": "frog" } } ``` ## Commenting out yq expressions Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly. Given a sample.yaml file of: ```yaml a: b: old ``` And an 'update.yq' expression file of: ```bash #! yq # This is a yq expression that updates the map # for several great reasons outlined here. .a.b = "new" # line comment here # | .a.c = "frog" # Now good things will happen. ``` then ```bash yq --from-file update.yq sample.yml ``` will output ```yaml a: b: new ``` ================================================ FILE: pkg/yqlib/doc/usage/hcl.md ================================================ # HCL Encode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl). HCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support: - Blocks and attributes - String interpolation and expressions (preserved without quotes) - Comments (leading, head, and line comments) - Nested structures (maps and lists) - Syntax colorisation when enabled ## Parse HCL Given a sample.hcl file of: ```hcl io_mode = "async" ``` then ```bash yq -oy sample.hcl ``` will output ```yaml io_mode: "async" ``` ## Roundtrip: Sample Doc Given a sample.hcl file of: ```hcl service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ``` then ```bash yq sample.hcl ``` will output ```hcl service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ``` ## Roundtrip: With an update Given a sample.hcl file of: ```hcl service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ``` then ```bash yq '.service.cat.process.main.command += "meow"' sample.hcl ``` will output ```hcl service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server", "meow"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ``` ## Parse HCL: Sample Doc Given a sample.hcl file of: ```hcl service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ``` then ```bash yq -oy sample.hcl ``` will output ```yaml service: cat: process: main: command: - "/usr/local/bin/awesome-app" - "server" management: command: - "/usr/local/bin/awesome-app" - "management" ``` ## Parse HCL: with comments Given a sample.hcl file of: ```hcl # Configuration port = 8080 # server port ``` then ```bash yq -oy sample.hcl ``` will output ```yaml # Configuration port: 8080 # server port ``` ## Roundtrip: with comments Given a sample.hcl file of: ```hcl # Configuration port = 8080 ``` then ```bash yq sample.hcl ``` will output ```hcl # Configuration port = 8080 ``` ## Roundtrip: With templates, functions and arithmetic Given a sample.hcl file of: ```hcl # Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message) ``` then ```bash yq sample.hcl ``` will output ```hcl # Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message) ``` ## Roundtrip: Separate blocks with same name. Given a sample.hcl file of: ```hcl resource "aws_instance" "web" { ami = "ami-12345" } resource "aws_instance" "db" { ami = "ami-67890" } ``` then ```bash yq sample.hcl ``` will output ```hcl resource "aws_instance" "web" { ami = "ami-12345" } resource "aws_instance" "db" { ami = "ami-67890" } ``` ================================================ FILE: pkg/yqlib/doc/usage/headers/base64.md ================================================ # Base64 Encode and decode to and from Base64. Base64 assumes [RFC4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a UTF-8 string and not binary content. See below for examples ================================================ FILE: pkg/yqlib/doc/usage/headers/convert.md ================================================ # JSON Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON). Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling. ================================================ FILE: pkg/yqlib/doc/usage/headers/csv-tsv.md ================================================ # CSV Encode/Decode/Roundtrip CSV and TSV files. ## Encode Currently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: ```yaml - name: Bobo type: dog - name: Fifi type: cat ``` As well as arrays of arrays of scalars (strings/numbers/booleans): ```yaml - [Bobo, dog] - [Fifi, cat] ``` ## Decode Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries. The data will be coded into an array of objects, using the header rows as keys. ```csv name,type Bobo,dog Fifi,cat ``` ================================================ FILE: pkg/yqlib/doc/usage/headers/formatting-expressions.md ================================================ # Formatting Expressions `From version v4.41+` You can put expressions into `.yq` files, use whitespace and comments to break up complex expressions and explain what's going on. ================================================ FILE: pkg/yqlib/doc/usage/headers/hcl.md ================================================ # HCL Encode and decode to and from [HashiCorp Configuration Language (HCL)](https://github.com/hashicorp/hcl). HCL is commonly used in HashiCorp tools like Terraform for configuration files. The yq HCL encoder and decoder support: - Blocks and attributes - String interpolation and expressions (preserved without quotes) - Comments (leading, head, and line comments) - Nested structures (maps and lists) - Syntax colorisation when enabled ================================================ FILE: pkg/yqlib/doc/usage/headers/kyaml.md ================================================ # KYaml Encode and decode to and from KYaml (a restricted subset of YAML that uses flow-style collections). KYaml is useful when you want YAML data rendered in a compact, JSON-like form while still supporting YAML features like comments. Notes: - Strings are always double-quoted in KYaml output. - Anchors and aliases are expanded (KYaml output does not emit them). ================================================ FILE: pkg/yqlib/doc/usage/headers/properties.md ================================================ # Properties Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across. By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these. ================================================ FILE: pkg/yqlib/doc/usage/headers/recipes.md ================================================ # Recipes These examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation. Please see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator. ================================================ FILE: pkg/yqlib/doc/usage/headers/toml.md ================================================ # TOML Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip) ================================================ FILE: pkg/yqlib/doc/usage/headers/xml.md ================================================ # XML Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are. Consecutive xml nodes with the same name are assumed to be arrays. XML content data, attributes processing instructions and directives are all created as plain fields. This can be controlled by: | Flag | Default |Sample XML | | -- | -- | -- | | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | | `--xml-content-name` | `+content` | Meow in ```Meow true``` | | `--xml-directive-name` | `+directive` | `````` | | `--xml-proc-inst-prefix` | `+p_` | `````` | {% hint style="warning" %} Default Attribute Prefix will be changing in v4.30! In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@". This will affect users that have not set their own prefix and are not roundtripping XML changes. {% endhint %} ## Encoder / Decoder flag options In addition to the above flags, there are the following xml encoder/decoder options controlled by flags: | Flag | Default | Description | | -- | -- | -- | | `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| | `--xml-keep-namespace` | true | Keeps the namespace of attributes | | `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. | | `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` | | `--xml-skip-directives` | false | Skips over directives, e.g. `````` | See below for examples ================================================ FILE: pkg/yqlib/doc/usage/kyaml.md ================================================ # KYaml Encode and decode to and from KYaml (a restricted subset of YAML that uses flow-style collections). KYaml is useful when you want YAML data rendered in a compact, JSON-like form while still supporting YAML features like comments. Notes: - Strings are always double-quoted in KYaml output. - Anchors and aliases are expanded (KYaml output does not emit them). ## Encode kyaml: plain string scalar Strings are always double-quoted in KYaml output. Given a sample.yml file of: ```yaml cat ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml "cat" ``` ## encode flow mapping and sequence Given a sample.yml file of: ```yaml a: b c: - d ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { a: "b", c: [ "d", ], } ``` ## encode non-string scalars Given a sample.yml file of: ```yaml a: 12 b: true c: null d: "true" ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { a: 12, b: true, c: null, d: "true", } ``` ## quote non-identifier keys Given a sample.yml file of: ```yaml "1a": b "has space": c ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { "1a": "b", "has space": "c", } ``` ## escape quoted strings Given a sample.yml file of: ```yaml a: "line1\nline2\t\"q\"" ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { a: "line1\nline2\t\"q\"", } ``` ## preserve comments when encoding Given a sample.yml file of: ```yaml # leading a: 1 # a line # head b b: 2 c: # head d - d # d line - e # trailing ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml # leading { a: 1, # a line # head b b: 2, c: [ # head d "d", # d line "e", ], # trailing } ``` ## Encode kyaml: anchors and aliases KYaml output does not support anchors/aliases; they are expanded to concrete values. Given a sample.yml file of: ```yaml base: &base a: b copy: *base ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { base: { a: "b", }, copy: { a: "b", }, } ``` ## Encode kyaml: yaml to kyaml shows formatting differences KYaml uses flow-style collections (braces/brackets) and explicit commas. Given a sample.yml file of: ```yaml person: name: John pets: - cat - dog ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml { person: { name: "John", pets: [ "cat", "dog", ], }, } ``` ## Encode kyaml: nested lists of objects Lists and objects can be nested arbitrarily; KYaml always uses flow-style collections. Given a sample.yml file of: ```yaml - name: a items: - id: 1 tags: - k: x v: y - k: x2 v: y2 - id: 2 tags: - k: z v: w ``` then ```bash yq -o=kyaml '.' sample.yml ``` will output ```yaml [ { name: "a", items: [ { id: 1, tags: [ { k: "x", v: "y", }, { k: "x2", v: "y2", }, ], }, { id: 2, tags: [ { k: "z", v: "w", }, ], }, ], }, ] ``` ================================================ FILE: pkg/yqlib/doc/usage/lua.md ================================================ ## Basic input example Given a sample.lua file of: ```lua return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; ``` then ```bash yq -oy '.' sample.lua ``` will output ```yaml country: Australia cities: - Sydney - Melbourne - Brisbane - Perth ``` ## Basic output example Given a sample.yml file of: ```yaml --- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth ``` then ```bash yq -o=lua '.' sample.yml ``` will output ```lua return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; ``` ## Unquoted keys Uses the `--lua-unquoted` option to produce a nicer-looking output. Given a sample.yml file of: ```yaml --- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth ``` then ```bash yq -o=lua --lua-unquoted '.' sample.yml ``` will output ```lua return { country = "Australia"; -- this place cities = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; ``` ## Globals Uses the `--lua-globals` option to export the values into the global scope. Given a sample.yml file of: ```yaml --- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth ``` then ```bash yq -o=lua --lua-globals '.' sample.yml ``` will output ```lua country = "Australia"; -- this place cities = { "Sydney", "Melbourne", "Brisbane", "Perth", }; ``` ## Elaborate example Given a sample.yml file of: ```yaml --- hello: world tables: like: this keys: values ? look: non-string keys : True numbers: - decimal: 12345 - hex: 0x7fabc123 - octal: 0o30 - float: 123.45 - infinity: .inf plus_infinity: +.inf minus_infinity: -.inf - not: .nan ``` then ```bash yq -o=lua '.' sample.yml ``` will output ```lua return { ["hello"] = "world"; ["tables"] = { ["like"] = "this"; ["keys"] = "values"; [{ ["look"] = "non-string keys"; }] = true; }; ["numbers"] = { { ["decimal"] = 12345; }, { ["hex"] = 0x7fabc123; }, { ["octal"] = 24; }, { ["float"] = 123.45; }, { ["infinity"] = (1/0); ["plus_infinity"] = (1/0); ["minus_infinity"] = (-1/0); }, { ["not"] = (0/0); }, }; }; ``` ================================================ FILE: pkg/yqlib/doc/usage/properties.md ================================================ # Properties Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across. By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these. ## Encode properties Note that empty arrays and maps are not encoded by default. Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props sample.yml ``` will output ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ``` ## Encode properties with array brackets Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot). Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props --properties-array-brackets sample.yml ``` will output ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets[0] = cat person.pets[1].nested[0] = list entry person.food[0] = pizza ``` ## Encode properties - custom separator Use the --properties-separator flag to specify your own key/value separator. Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props --properties-separator=" :@ " sample.yml ``` will output ```properties # block comments come through # comments on values appear person.name :@ Mike Wazowski # comments on array values appear person.pets.0 :@ cat person.pets.1.nested.0 :@ list entry person.food.0 :@ pizza ``` ## Encode properties: scalar encapsulation Note that string values with blank characters in them are encapsulated with double quotes Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props --unwrapScalar=false sample.yml ``` will output ```properties # block comments come through # comments on values appear person.name = "Mike Wazowski" # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = "list entry" person.food.0 = pizza ``` ## Encode properties: no comments Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props '... comments = ""' sample.yml ``` will output ```properties person.name = Mike Wazowski person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ``` ## Encode properties: include empty maps and arrays Use a yq expression to set the empty maps and sequences to your desired value. Given a sample.yml file of: ```yaml # block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ``` then ```bash yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ""' sample.yml ``` will output ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza emptyArray = emptyMap = ``` ## Decode properties Given a sample.properties file of: ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ``` then ```bash yq -p=props sample.properties ``` will output ```yaml person: # block comments come through # comments on values appear name: Mike Wazowski pets: # comments on array values appear - cat - nested: - list entry food: - pizza ``` ## Decode properties: numbers All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type. Given a sample.properties file of: ```properties a.b = 10 ``` then ```bash yq -p=props ' (.. | select(tag == "!!str")) |= from_yaml' sample.properties ``` will output ```yaml a: b: 10 ``` ## Decode properties - array should be a map If you have a numeric map key in your property files, use array_to_map to convert them to maps. Given a sample.properties file of: ```properties things.10 = mike ``` then ```bash yq -p=props '.things |= array_to_map' sample.properties ``` will output ```yaml things: 10: mike ``` ## Roundtrip Given a sample.properties file of: ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ``` then ```bash yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties ``` will output ```properties # block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = dog person.pets.1.nested.0 = list entry person.food.0 = pizza ``` ================================================ FILE: pkg/yqlib/doc/usage/recipes.md ================================================ # Recipes These examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation. Please see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator. ## Find items in an array We have an array and we want to find the elements with a particular name. Given a sample.yml file of: ```yaml - name: Foo numBuckets: 0 - name: Bar numBuckets: 0 ``` then ```bash yq '.[] | select(.name == "Foo")' sample.yml ``` will output ```yaml name: Foo numBuckets: 0 ``` ### Explanation: - `.[]` splats the array, and puts all the items in the context. - These items are then piped (`|`) into `select(.name == "Foo")` which will select all the nodes that have a name property set to 'Foo'. - See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information. ## Find and update items in an array We have an array and we want to _update_ the elements with a particular name. Given a sample.yml file of: ```yaml - name: Foo numBuckets: 0 - name: Bar numBuckets: 0 ``` then ```bash yq '(.[] | select(.name == "Foo") | .numBuckets) |= . + 1' sample.yml ``` will output ```yaml - name: Foo numBuckets: 1 - name: Bar numBuckets: 0 ``` ### Explanation: - Following from the example above`.[]` splats the array, selects filters the items. - We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items - Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression. - `|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`). - The expression `. + 1` increments the numBuckets counter. - See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information. ## Deeply prune a tree Say we are only interested in child1 and child2, and want to filter everything else out. Given a sample.yml file of: ```yaml parentA: - bob parentB: child1: i am child1 child3: hiya parentC: childX: cool child2: me child2 ``` then ```bash yq '( .. | # recurse through all the nodes select(has("child1") or has("child2")) | # match parents that have either child1 or child2 (.child1, .child2) | # select those children select(.) # filter out nulls ) as $i ireduce({}; # using that set of nodes, create a new result map setpath($i | path; $i) # and put in each node, using its original path )' sample.yml ``` will output ```yaml parentB: child1: i am child1 parentC: child2: me child2 ``` ### Explanation: - Find all the matching child1 and child2 nodes - Using ireduce, create a new map using just those nodes - Set each node into the new map using its original path ## Multiple or complex updates to items in an array We have an array and we want to _update_ the elements with a particular name in reference to its type. Given a sample.yml file of: ```yaml myArray: - name: Foo type: cat - name: Bar type: dog ``` then ```bash yq 'with(.myArray[]; .name = .name + " - " + .type)' sample.yml ``` will output ```yaml myArray: - name: Foo - cat type: cat - name: Bar - dog type: dog ``` ### Explanation: - The with operator will effectively loop through each given item in the first given expression, and run the second expression against it. - `.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array - `.name = .name + " - " + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type. - See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples. ## Sort an array by a field Given a sample.yml file of: ```yaml myArray: - name: Foo numBuckets: 1 - name: Bar numBuckets: 0 ``` then ```bash yq '.myArray |= sort_by(.numBuckets)' sample.yml ``` will output ```yaml myArray: - name: Bar numBuckets: 0 - name: Foo numBuckets: 1 ``` ### Explanation: - We want to resort `.myArray`. - `sort_by` works by piping an array into it, and it pipes out a sorted array. - So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))` ## Filter, flatten, sort and unique Lets find the unique set of names from the document. Given a sample.yml file of: ```yaml - type: foo names: - Fred - Catherine - type: bar names: - Zelda - type: foo names: Fred - type: foo names: Ava ``` then ```bash yq '[.[] | select(.type == "foo") | .names] | flatten | sort | unique' sample.yml ``` will output ```yaml - Ava - Catherine - Fred ``` ### Explanation: - `.[] | select(.type == "foo") | .names` will select the array elements of type "foo" - Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator. - But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == "foo") | .names]` - Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own. - Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings - Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names! - See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples. ## Export as environment variables (script), or any custom format Given a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats. Given a sample.yml file of: ```yaml var0: string0 var1: string1 fruit: - apple - banana - peach ``` then ```bash yq '.[] |( ( select(kind == "scalar") | key + "='\''" + . + "'\''"), ( select(kind == "seq") | key + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")") )' sample.yml ``` will output ```yaml var0='string0' var1='string1' fruit=('apple','banana','peach') ``` ### Explanation: - `.[]` matches all top level elements - We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together - Scalars, we just need the key and quoted value: `( select(kind == "scalar") | key + "='" + . + "'")` - Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map("'" + . + "'") | join(",")` ## Custom format with nested data Like the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them. Given a sample.yml file of: ```yaml simple: string0 simpleArray: - apple - banana - peach deep: property: value array: - cat ``` then ```bash yq '.. |( ( select(kind == "scalar" and parent | kind != "seq") | (path | join("_")) + "='\''" + . + "'\''"), ( select(kind == "seq") | (path | join("_")) + "=(" + (map("'\''" + . + "'\''") | join(",")) + ")") )' sample.yml ``` will output ```yaml simple='string0' deep_property='value' simpleArray=('apple','banana','peach') deep_array=('cat') ``` ### Explanation: - You'll need to understand how the previous example works to understand this extension. - `..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements. - Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together - This time, however, our expression matches every node in the data structure. - We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != "seq"` to the select operator expression for scalars - We don't just want the key any more, we want the full path. So instead of `key` we have `path | join("_")` - The expression for sequences follows the same logic ================================================ FILE: pkg/yqlib/doc/usage/shellvariables.md ================================================ ## Encode shell variables Note that comments are dropped and values will be enclosed in single quotes as needed. Given a sample.yml file of: ```yaml # comment name: Mike Wazowski eyes: color: turquoise number: 1 friends: - James P. Sullivan - Celia Mae ``` then ```bash yq -o=shell sample.yml ``` will output ```sh name='Mike Wazowski' eyes_color=turquoise eyes_number=1 friends_0='James P. Sullivan' friends_1='Celia Mae' ``` ## Encode shell variables: illegal variable names as key. Keys that would be illegal as variable keys are adapted. Given a sample.yml file of: ```yaml ascii_=_symbols: replaced with _ "ascii_ _controls": dropped (this example uses \t) nonascii_א_characters: dropped effort_expeñded_tò_preserve_accented_latin_letters: moderate (via unicode NFKD) ``` then ```bash yq -o=shell sample.yml ``` will output ```sh ascii___symbols='replaced with _' ascii__controls='dropped (this example uses \t)' nonascii__characters=dropped effort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)' ``` ## Encode shell variables: empty values, arrays and maps Empty values are encoded to empty variables, but empty arrays and maps are skipped. Given a sample.yml file of: ```yaml empty: value: array: [] map: {} ``` then ```bash yq -o=shell sample.yml ``` will output ```sh empty_value= ``` ## Encode shell variables: single quotes in values Single quotes in values are encoded as '"'"' (close single quote, double-quoted single quote, open single quote). Given a sample.yml file of: ```yaml name: Miles O'Brien ``` then ```bash yq -o=shell sample.yml ``` will output ```sh name='Miles O'"'"'Brien' ``` ## Encode shell variables: custom separator Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores. Given a sample.yml file of: ```yaml my_app: db_config: host: localhost port: 5432 ``` then ```bash yq -o=shell --shell-key-separator="__" sample.yml ``` will output ```sh my_app__db_config__host=localhost my_app__db_config__port=5432 ``` ================================================ FILE: pkg/yqlib/doc/usage/toml.md ================================================ # TOML Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip) ## Parse: Simple Given a sample.toml file of: ```toml A = "hello" B = 12 ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml A: hello B: 12 ``` ## Parse: Deep paths Given a sample.toml file of: ```toml person.name = "hello" person.address = "12 cat st" ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml person: name: hello address: 12 cat st ``` ## Encode: Scalar Given a sample.toml file of: ```toml person.name = "hello" person.address = "12 cat st" ``` then ```bash yq '.person.name' sample.toml ``` will output ```yaml hello ``` ## Parse: inline table Given a sample.toml file of: ```toml name = { first = "Tom", last = "Preston-Werner" } ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml name: first: Tom last: Preston-Werner ``` ## Parse: Array Table Given a sample.toml file of: ```toml [owner.contact] name = "Tom Preston-Werner" age = 36 [[owner.addresses]] street = "first street" suburb = "ok" [[owner.addresses]] street = "second street" suburb = "nice" ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml owner: contact: name: Tom Preston-Werner age: 36 addresses: - street: first street suburb: ok - street: second street suburb: nice ``` ## Parse: Array of Array Table Given a sample.toml file of: ```toml [[fruits]] name = "apple" [[fruits.varieties]] # nested array of tables name = "red delicious" ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml fruits: - name: apple varieties: - name: red delicious ``` ## Parse: Empty Table Given a sample.toml file of: ```toml [dependencies] ``` then ```bash yq -oy '.' sample.toml ``` will output ```yaml dependencies: {} ``` ## Roundtrip: inline table attribute Given a sample.toml file of: ```toml name = { first = "Tom", last = "Preston-Werner" } ``` then ```bash yq '.' sample.toml ``` will output ```yaml name = { first = "Tom", last = "Preston-Werner" } ``` ## Roundtrip: table section Given a sample.toml file of: ```toml [owner.contact] name = "Tom" age = 36 ``` then ```bash yq '.' sample.toml ``` will output ```yaml [owner.contact] name = "Tom" age = 36 ``` ## Roundtrip: array of tables Given a sample.toml file of: ```toml [[fruits]] name = "apple" [[fruits.varieties]] name = "red delicious" ``` then ```bash yq '.' sample.toml ``` will output ```yaml [[fruits]] name = "apple" [[fruits.varieties]] name = "red delicious" ``` ## Roundtrip: arrays and scalars Given a sample.toml file of: ```toml A = ["hello", ["world", "again"]] B = 12 ``` then ```bash yq '.' sample.toml ``` will output ```yaml A = ["hello", ["world", "again"]] B = 12 ``` ## Roundtrip: simple Given a sample.toml file of: ```toml A = "hello" B = 12 ``` then ```bash yq '.' sample.toml ``` will output ```yaml A = "hello" B = 12 ``` ## Roundtrip: deep paths Given a sample.toml file of: ```toml [person] name = "hello" address = "12 cat st" ``` then ```bash yq '.' sample.toml ``` will output ```yaml [person] name = "hello" address = "12 cat st" ``` ## Roundtrip: empty array Given a sample.toml file of: ```toml A = [] ``` then ```bash yq '.' sample.toml ``` will output ```yaml A = [] ``` ## Roundtrip: sample table Given a sample.toml file of: ```toml var = "x" [owner.contact] name = "Tom Preston-Werner" age = 36 ``` then ```bash yq '.' sample.toml ``` will output ```yaml var = "x" [owner.contact] name = "Tom Preston-Werner" age = 36 ``` ## Roundtrip: empty table Given a sample.toml file of: ```toml [dependencies] ``` then ```bash yq '.' sample.toml ``` will output ```yaml [dependencies] ``` ## Roundtrip: comments Given a sample.toml file of: ```toml # This is a comment A = "hello" # inline comment B = 12 # Table comment [person] name = "Tom" # name comment ``` then ```bash yq '.' sample.toml ``` will output ```yaml # This is a comment A = "hello" # inline comment B = 12 # Table comment [person] name = "Tom" # name comment ``` ## Roundtrip: sample from web Given a sample.toml file of: ```toml # This is a TOML document title = "TOML Example" [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 [database] enabled = true ports = [8000, 8001, 8002] data = [["delta", "phi"], [3.14]] temp_targets = { cpu = 79.5, case = 72.0 } # [servers] yq can't do this one yet [servers.alpha] ip = "10.0.0.1" role = "frontend" [servers.beta] ip = "10.0.0.2" role = "backend" ``` then ```bash yq '.' sample.toml ``` will output ```yaml # This is a TOML document title = "TOML Example" [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 [database] enabled = true ports = [8000, 8001, 8002] data = [["delta", "phi"], [3.14]] temp_targets = { cpu = 79.5, case = 72.0 } # [servers] yq can't do this one yet [servers.alpha] ip = "10.0.0.1" role = "frontend" [servers.beta] ip = "10.0.0.2" role = "backend" ``` ================================================ FILE: pkg/yqlib/doc/usage/xml.md ================================================ # XML Encode and decode to and from XML. Whitespace is not conserved for round trips - but the order of the fields are. Consecutive xml nodes with the same name are assumed to be arrays. XML content data, attributes processing instructions and directives are all created as plain fields. This can be controlled by: | Flag | Default |Sample XML | | -- | -- | -- | | `--xml-attribute-prefix` | `+` (changing to `+@` soon) | Legs in `````` | | `--xml-content-name` | `+content` | Meow in ```Meow true``` | | `--xml-directive-name` | `+directive` | `````` | | `--xml-proc-inst-prefix` | `+p_` | `````` | {% hint style="warning" %} Default Attribute Prefix will be changing in v4.30! In order to avoid name conflicts (e.g. having an attribute named "content" will create a field that clashes with the default content name of "+content") the attribute prefix will be changing to "+@". This will affect users that have not set their own prefix and are not roundtripping XML changes. {% endhint %} ## Encoder / Decoder flag options In addition to the above flags, there are the following xml encoder/decoder options controlled by flags: | Flag | Default | Description | | -- | -- | -- | | `--xml-strict-mode` | false | Strict mode enforces the requirements of the XML specification. When switched off the parser allows input containing common mistakes. See [the Golang xml decoder ](https://pkg.go.dev/encoding/xml#Decoder) for more details.| | `--xml-keep-namespace` | true | Keeps the namespace of attributes | | `--xml-raw-token` | true | Does not verify that start and end elements match and does not translate name space prefixes to their corresponding URLs. | | `--xml-skip-proc-inst` | false | Skips over processing instructions, e.g. `` | | `--xml-skip-directives` | false | Skips over directives, e.g. `````` | See below for examples ## Parse xml: simple Notice how all the values are strings, see the next example on how you can fix that. Given a sample.xml file of: ```xml meow 4 true ``` then ```bash yq -oy sample.xml ``` will output ```yaml +p_xml: version="1.0" encoding="UTF-8" cat: says: meow legs: "4" cute: "true" ``` ## Parse xml: number All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type. Given a sample.xml file of: ```xml meow 4 true ``` then ```bash yq -oy ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml ``` will output ```yaml +p_xml: version="1.0" encoding="UTF-8" cat: says: meow legs: 4 cute: true ``` ## Parse xml: array Consecutive nodes with identical xml names are assumed to be arrays. Given a sample.xml file of: ```xml cat goat ``` then ```bash yq -oy sample.xml ``` will output ```yaml +p_xml: version="1.0" encoding="UTF-8" animal: - cat - goat ``` ## Parse xml: force as an array In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple. Given a sample.xml file of: ```xml cat ``` then ```bash yq -oy '.zoo.animal |= ([] + .)' sample.xml ``` will output ```yaml zoo: animal: - cat ``` ## Parse xml: force all as an array Given a sample.xml file of: ```xml boing ``` then ```bash yq -oy '.. |= [] + .' sample.xml ``` will output ```yaml - zoo: - thing: - frog: - boing ``` ## Parse xml: attributes Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own. Given a sample.xml file of: ```xml 7 ``` then ```bash yq -oy sample.xml ``` will output ```yaml +p_xml: version="1.0" encoding="UTF-8" cat: +@legs: "4" legs: "7" ``` ## Parse xml: attributes with content Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own. Given a sample.xml file of: ```xml meow ``` then ```bash yq -oy sample.xml ``` will output ```yaml +p_xml: version="1.0" encoding="UTF-8" cat: +content: meow +@legs: "4" ``` ## Parse xml: content split between comments/children Multiple content texts are collected into a sequence. Given a sample.xml file of: ```xml value anotherValue frog cool! ``` then ```bash yq -oy sample.xml ``` will output ```yaml root: +content: # comment - value - anotherValue - cool! a: frog ``` ## Parse xml: custom dtd DTD entities are processed as directives. Given a sample.xml file of: ```xml ]> &writer;©right; ``` then ```bash yq sample.xml ``` will output ```xml ]> &writer;&copyright; ``` ## Parse xml: skip custom dtd DTDs are directives, skip over directives to skip DTDs. Given a sample.xml file of: ```xml ]> &writer;©right; ``` then ```bash yq --xml-skip-directives sample.xml ``` will output ```xml &writer;&copyright; ``` ## Parse xml: with comments A best attempt is made to preserve comments. Given a sample.xml file of: ```xml 3 z ``` then ```bash yq -oy sample.xml ``` will output ```yaml # before cat cat: # in cat before x: "3" # multi # line comment # for x # before y y: # in y before # in d before d: z # in d after # in y after # in_cat_after # after cat ``` ## Parse xml: keep attribute namespace Defaults to true Given a sample.xml file of: ```xml baz foobar ``` then ```bash yq --xml-keep-namespace=false sample.xml ``` will output ```xml baz foobar ``` instead of ```xml baz foobar ``` ## Parse xml: keep raw attribute namespace Defaults to true Given a sample.xml file of: ```xml baz foobar ``` then ```bash yq --xml-raw-token=false sample.xml ``` will output ```xml baz foobar ``` instead of ```xml baz foobar ``` ## Encode xml: simple Given a sample.yml file of: ```yaml cat: purrs ``` then ```bash yq -o=xml sample.yml ``` will output ```xml purrs ``` ## Encode xml: array Given a sample.yml file of: ```yaml pets: cat: - purrs - meows ``` then ```bash yq -o=xml sample.yml ``` will output ```xml purrs meows ``` ## Encode xml: attributes Fields with the matching xml-attribute-prefix are assumed to be attributes. Given a sample.yml file of: ```yaml cat: +@name: tiger meows: true ``` then ```bash yq -o=xml sample.yml ``` will output ```xml true ``` ## Encode xml: attributes with content Fields with the matching xml-content-name is assumed to be content. Given a sample.yml file of: ```yaml cat: +@name: tiger +content: cool ``` then ```bash yq -o=xml sample.yml ``` will output ```xml cool ``` ## Encode xml: comments A best attempt is made to copy comments to xml. Given a sample.yml file of: ```yaml # # header comment # above_cat # cat: # inline_cat # above_array array: # inline_array - val1 # inline_val1 # above_val2 - val2 # inline_val2 # below_cat ``` then ```bash yq -o=xml sample.yml ``` will output ```xml val1 val2 ``` ## Encode: doctype and xml declaration Use the special xml names to add/modify proc instructions and directives. Given a sample.yml file of: ```yaml +p_xml: version="1.0" +directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" ' apple: +p_coolioo: version="1.0" +directive: 'CATYPE meow purr puss ' b: things ``` then ```bash yq -o=xml sample.yml ``` will output ```xml things ``` ## Round trip: with comments A best effort is made, but comment positions and white space are not preserved perfectly. Given a sample.xml file of: ```xml 3 z ``` then ```bash yq sample.xml ``` will output ```xml 3 z ``` ## Roundtrip: with doctype and declaration yq parses XML proc instructions and directives into nodes. Unfortunately the underlying XML parser loses whitespace information. Given a sample.xml file of: ```xml things ``` then ```bash yq sample.xml ``` will output ```xml things ``` ================================================ FILE: pkg/yqlib/encoder.go ================================================ package yqlib import ( "bufio" "errors" "io" "strings" "github.com/fatih/color" ) type Encoder interface { Encode(writer io.Writer, node *CandidateNode) error PrintDocumentSeparator(writer io.Writer) error PrintLeadingContent(writer io.Writer, content string) error CanHandleAliases() bool } func mapKeysToStrings(node *CandidateNode) { if node.Kind == MappingNode { for index, child := range node.Content { if index%2 == 0 { // its a map key child.Tag = "!!str" } } } for _, child := range node.Content { mapKeysToStrings(child) } } // Some funcs are shared between encoder_yaml and encoder_kyaml func PrintYAMLDocumentSeparator(writer io.Writer, PrintDocSeparators bool) error { if PrintDocSeparators { log.Debug("writing doc sep") if err := writeString(writer, "---\n"); err != nil { return err } } return nil } func PrintYAMLLeadingContent(writer io.Writer, content string, PrintDocSeparators bool, ColorsEnabled bool) error { reader := bufio.NewReader(strings.NewReader(content)) // reuse precompiled package-level regex // (declared in decoder_yaml.go) for { readline, errReading := reader.ReadString('\n') if errReading != nil && !errors.Is(errReading, io.EOF) { return errReading } if strings.Contains(readline, "$yqDocSeparator$") { // Preserve the original line ending (CRLF or LF) lineEnding := "\n" if strings.HasSuffix(readline, "\r\n") { lineEnding = "\r\n" } if PrintDocSeparators { if err := writeString(writer, "---"+lineEnding); err != nil { return err } } } else { if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRe.MatchString(readline) { readline = "# " + readline } if ColorsEnabled && strings.TrimSpace(readline) != "" { readline = format(color.FgHiBlack) + readline + format(color.Reset) } if err := writeString(writer, readline); err != nil { return err } } if errors.Is(errReading, io.EOF) { if readline != "" { // the last comment we read didn't have a newline, put one in if err := writeString(writer, "\n"); err != nil { return err } } break } } return nil } ================================================ FILE: pkg/yqlib/encoder_base64.go ================================================ //go:build !yq_nobase64 package yqlib import ( "encoding/base64" "fmt" "io" ) type base64Encoder struct { encoding base64.Encoding } func NewBase64Encoder() Encoder { return &base64Encoder{encoding: *base64.StdEncoding} } func (e *base64Encoder) CanHandleAliases() bool { return false } func (e *base64Encoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error { if node.guessTagFromCustomType() != "!!str" { return fmt.Errorf("cannot encode %v as base64, can only operate on strings", node.Tag) } _, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value)))) return err } ================================================ FILE: pkg/yqlib/encoder_csv.go ================================================ //go:build !yq_nocsv package yqlib import ( "encoding/csv" "fmt" "io" ) type csvEncoder struct { separator rune } func NewCsvEncoder(prefs CsvPreferences) Encoder { return &csvEncoder{separator: prefs.Separator} } func (e *csvEncoder) CanHandleAliases() bool { return false } func (e *csvEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (e *csvEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*CandidateNode) error { stringValues := make([]string, len(contents)) for i, child := range contents { if child.Kind != ScalarNode { return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) } stringValues[i] = child.Value } return csvWriter.Write(stringValues) } func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*CandidateNode) error { for i, child := range content { if child.Kind != SequenceNode { return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag) } err := e.encodeRow(csvWriter, child.Content) if err != nil { return err } } return nil } func (e *csvEncoder) extractHeader(child *CandidateNode) ([]*CandidateNode, error) { if child.Kind != MappingNode { return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag) } mapKeys := getMapKeys(child) return mapKeys.Content, nil } func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNode) []*CandidateNode { childRow := make([]*CandidateNode, 0) for _, header := range headers { keyIndex := findKeyInMap(child, header) value := createScalarNode(nil, "") if keyIndex != -1 { value = child.Content[keyIndex+1] } childRow = append(childRow, value) } return childRow } func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNode) error { headers, err := e.extractHeader(content[0]) if err != nil { return nil } err = e.encodeRow(csvWriter, headers) if err != nil { return nil } for i, child := range content { if child.Kind != MappingNode { return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag) } row := e.createChildRow(child, headers) err = e.encodeRow(csvWriter, row) if err != nil { return err } } return nil } func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.Kind == ScalarNode { return writeString(writer, node.Value+"\n") } csvWriter := csv.NewWriter(writer) csvWriter.Comma = e.separator // node must be a sequence if node.Kind != SequenceNode { return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag) } else if len(node.Content) == 0 { return nil } if node.Content[0].Kind == ScalarNode { return e.encodeRow(csvWriter, node.Content) } if node.Content[0].Kind == MappingNode { return e.encodeObjects(csvWriter, node.Content) } return e.encodeArrays(csvWriter, node.Content) } ================================================ FILE: pkg/yqlib/encoder_hcl.go ================================================ //go:build !yq_nohcl package yqlib import ( "fmt" "io" "regexp" "strings" "github.com/fatih/color" "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" hclwrite "github.com/hashicorp/hcl/v2/hclwrite" "github.com/zclconf/go-cty/cty" ) type hclEncoder struct { prefs HclPreferences } // commentPathSep is used to join path segments when collecting comments. // It uses a rarely used ASCII control character to avoid collisions with // normal key names (including dots). const commentPathSep = "\x1e" // NewHclEncoder creates a new HCL encoder func NewHclEncoder(prefs HclPreferences) Encoder { return &hclEncoder{prefs: prefs} } func (he *hclEncoder) CanHandleAliases() bool { return false } func (he *hclEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (he *hclEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debugf("I need to encode %v", NodeToString(node)) if node.Kind == ScalarNode { return writeString(writer, node.Value+"\n") } f := hclwrite.NewEmptyFile() body := f.Body() // Collect comments as we encode commentMap := make(map[string]string) he.collectComments(node, "", commentMap) if err := he.encodeNode(body, node); err != nil { return fmt.Errorf("failed to encode HCL: %w", err) } // Get the formatted output and remove extra spacing before '=' output := f.Bytes() compactOutput := he.compactSpacing(output) // Inject comments back into the output finalOutput := he.injectComments(compactOutput, commentMap) if he.prefs.ColorsEnabled { colourized := he.colorizeHcl(finalOutput) _, err := writer.Write(colourized) return err } _, err := writer.Write(finalOutput) return err } // compactSpacing removes extra whitespace before '=' in attribute assignments func (he *hclEncoder) compactSpacing(input []byte) []byte { // Use regex to replace multiple spaces before = with single space re := regexp.MustCompile(`(\S)\s{2,}=`) return re.ReplaceAll(input, []byte("$1 =")) } // collectComments recursively collects comments from nodes for later injection func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commentMap map[string]string) { if node == nil { return } // For mapping nodes, collect comments from keys and values if node.Kind == MappingNode { // Collect root-level head comment if at root (prefix is empty) if prefix == "" && node.HeadComment != "" { commentMap[joinCommentPath("__root__", "head")] = node.HeadComment } for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] key := keyNode.Value // Create a path for this key path := joinCommentPath(prefix, key) // Store comments from the key (head comments appear before the attribute) if keyNode.HeadComment != "" { commentMap[joinCommentPath(path, "head")] = keyNode.HeadComment } // Store comments from the value (line comments appear after the value) if valueNode.LineComment != "" { commentMap[joinCommentPath(path, "line")] = valueNode.LineComment } if valueNode.FootComment != "" { commentMap[joinCommentPath(path, "foot")] = valueNode.FootComment } // Recurse into nested mappings if valueNode.Kind == MappingNode { he.collectComments(valueNode, path, commentMap) } } } } // joinCommentPath concatenates path segments using commentPathSep, safely handling empty prefixes. func joinCommentPath(prefix, segment string) string { if prefix == "" { return segment } return prefix + commentPathSep + segment } // injectComments adds collected comments back into the HCL output func (he *hclEncoder) injectComments(output []byte, commentMap map[string]string) []byte { // Convert output to string for easier manipulation result := string(output) // Root-level head comment (stored on the synthetic __root__/head path) for path, comment := range commentMap { if path == joinCommentPath("__root__", "head") { trimmed := strings.TrimSpace(comment) if trimmed != "" && !strings.HasPrefix(result, trimmed) { result = trimmed + "\n" + result } } } // Attribute head comments: insert above matching assignment for path, comment := range commentMap { parts := strings.Split(path, commentPathSep) if len(parts) < 2 { continue } commentType := parts[len(parts)-1] key := parts[len(parts)-2] if commentType != "head" || key == "" { continue } trimmed := strings.TrimSpace(comment) if trimmed == "" { continue } re := regexp.MustCompile(`(?m)^(\s*)` + regexp.QuoteMeta(key) + `\s*=`) if re.MatchString(result) { result = re.ReplaceAllString(result, "$1"+trimmed+"\n$0") } } return []byte(result) } func (he *hclEncoder) colorizeHcl(input []byte) []byte { hcl := string(input) result := strings.Builder{} // Create colour functions for different token types commentColor := color.New(color.FgHiBlack).SprintFunc() stringColor := color.New(color.FgGreen).SprintFunc() numberColor := color.New(color.FgHiMagenta).SprintFunc() keyColor := color.New(color.FgCyan).SprintFunc() boolColor := color.New(color.FgHiMagenta).SprintFunc() // Simple tokenization for HCL colouring i := 0 for i < len(hcl) { ch := hcl[i] // Comments - from # to end of line if ch == '#' { end := i for end < len(hcl) && hcl[end] != '\n' { end++ } result.WriteString(commentColor(hcl[i:end])) i = end continue } // Strings - quoted text if ch == '"' || ch == '\'' { quote := ch end := i + 1 for end < len(hcl) && hcl[end] != quote { if hcl[end] == '\\' { end++ // skip escaped char } end++ } if end < len(hcl) { end++ // include closing quote } result.WriteString(stringColor(hcl[i:end])) i = end continue } // Numbers - sequences of digits, possibly with decimal point or minus if (ch >= '0' && ch <= '9') || (ch == '-' && i+1 < len(hcl) && hcl[i+1] >= '0' && hcl[i+1] <= '9') { end := i if ch == '-' { end++ } for end < len(hcl) && ((hcl[end] >= '0' && hcl[end] <= '9') || hcl[end] == '.') { end++ } result.WriteString(numberColor(hcl[i:end])) i = end continue } // Identifiers/keys - alphanumeric + underscore if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' { end := i for end < len(hcl) && ((hcl[end] >= 'a' && hcl[end] <= 'z') || (hcl[end] >= 'A' && hcl[end] <= 'Z') || (hcl[end] >= '0' && hcl[end] <= '9') || hcl[end] == '_' || hcl[end] == '-') { end++ } ident := hcl[i:end] // Check if this is a keyword/reserved word switch ident { case "true", "false", "null": result.WriteString(boolColor(ident)) default: // Check if followed by = (it's a key) j := end for j < len(hcl) && (hcl[j] == ' ' || hcl[j] == '\t') { j++ } if j < len(hcl) && hcl[j] == '=' { result.WriteString(keyColor(ident)) } else if j < len(hcl) && hcl[j] == '{' { // Block type result.WriteString(keyColor(ident)) } else { result.WriteString(ident) // plain text for other identifiers } } i = end continue } // Everything else (whitespace, operators, brackets) - no color result.WriteByte(ch) i++ } return []byte(result.String()) } // Helper runes for unquoted identifiers func isHCLIdentifierStart(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' } func isHCLIdentifierPart(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' || r == '-' } func isValidHCLIdentifier(s string) bool { if s == "" { return false } // HCL identifiers must start with a letter or underscore // and contain only letters, digits, underscores, and hyphens for i, r := range s { if i == 0 { if !isHCLIdentifierStart(r) { return false } continue } if !isHCLIdentifierPart(r) { return false } } return true } // tokensForRawHCLExpr produces a minimal token stream for a simple HCL expression so we can // write it without introducing quotes (e.g. function calls like upper(message)). func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) { var tokens hclwrite.Tokens for i := 0; i < len(expr); { ch := expr[i] switch { case ch == ' ' || ch == '\t': i++ continue case isHCLIdentifierStart(rune(ch)): start := i i++ for i < len(expr) && isHCLIdentifierPart(rune(expr[i])) { i++ } tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(expr[start:i])}) continue case ch >= '0' && ch <= '9': start := i i++ for i < len(expr) && ((expr[i] >= '0' && expr[i] <= '9') || expr[i] == '.') { i++ } tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenNumberLit, Bytes: []byte(expr[start:i])}) continue case ch == '(': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenOParen, Bytes: []byte{'('}}) case ch == ')': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCParen, Bytes: []byte{')'}}) case ch == ',': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenComma, Bytes: []byte{','}}) case ch == '.': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenDot, Bytes: []byte{'.'}}) case ch == '+': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenPlus, Bytes: []byte{'+'}}) case ch == '-': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenMinus, Bytes: []byte{'-'}}) case ch == '*': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenStar, Bytes: []byte{'*'}}) case ch == '/': tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenSlash, Bytes: []byte{'/'}}) default: return nil, fmt.Errorf("unsupported character %q in raw HCL expression", ch) } i++ } return tokens, nil } // encodeAttribute encodes a value as an HCL attribute func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode *CandidateNode) error { if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" { // Handle unquoted expressions (as-is, without quotes) if valueNode.Style == 0 { tokens, err := tokensForRawHCLExpr(valueNode.Value) if err != nil { return err } body.SetAttributeRaw(key, tokens) return nil } if valueNode.Style&LiteralStyle != 0 { tokens, err := tokensForRawHCLExpr(valueNode.Value) if err != nil { return err } body.SetAttributeRaw(key, tokens) return nil } // Check if template with interpolation if valueNode.Style&DoubleQuotedStyle != 0 && strings.Contains(valueNode.Value, "${") { return he.encodeTemplateAttribute(body, key, valueNode.Value) } // Check if unquoted identifier if isValidHCLIdentifier(valueNode.Value) && valueNode.Style == 0 { traversal := hcl.Traversal{ hcl.TraverseRoot{Name: valueNode.Value}, } body.SetAttributeTraversal(key, traversal) return nil } } // Default: use cty.Value for quoted strings and all other types ctyValue, err := nodeToCtyValue(valueNode) if err != nil { return err } body.SetAttributeValue(key, ctyValue) return nil } // encodeTemplateAttribute encodes a template string with ${} interpolations func (he *hclEncoder) encodeTemplateAttribute(body *hclwrite.Body, key string, templateStr string) error { tokens := hclwrite.Tokens{ {Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}}, } for i := 0; i < len(templateStr); i++ { if i < len(templateStr)-1 && templateStr[i] == '$' && templateStr[i+1] == '{' { // Start of template interpolation tokens = append(tokens, &hclwrite.Token{ Type: hclsyntax.TokenTemplateInterp, Bytes: []byte("${"), }) i++ // skip the '{' // Find the matching '}' start := i + 1 depth := 1 for i++; i < len(templateStr) && depth > 0; i++ { switch templateStr[i] { case '{': depth++ case '}': depth-- } } i-- // back up to the '}' interpExpr := templateStr[start:i] tokens = append(tokens, &hclwrite.Token{ Type: hclsyntax.TokenIdent, Bytes: []byte(interpExpr), }) tokens = append(tokens, &hclwrite.Token{ Type: hclsyntax.TokenTemplateSeqEnd, Bytes: []byte("}"), }) } else { // Regular character tokens = append(tokens, &hclwrite.Token{ Type: hclsyntax.TokenQuotedLit, Bytes: []byte{templateStr[i]}, }) } } tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}}) body.SetAttributeRaw(key, tokens) return nil } // encodeBlockIfMapping attempts to encode a value as a block. Returns true if it was encoded as a block. func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valueNode *CandidateNode) bool { if valueNode.Kind != MappingNode || valueNode.Style == FlowStyle { return false } // If EncodeSeparate is set, emit children as separate blocks regardless of label extraction if valueNode.EncodeSeparate { if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled { return true } } // Try to extract block labels from a single-entry mapping chain if labels, bodyNode, ok := extractBlockLabels(valueNode); ok { if len(labels) > 1 && mappingChildrenAllMappings(bodyNode) { primaryLabels := labels[:len(labels)-1] nestedType := labels[len(labels)-1] block := body.AppendNewBlock(key, primaryLabels) if handled, err := he.encodeMappingChildrenAsBlocks(block.Body(), nestedType, bodyNode); err == nil && handled { return true } if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil { return true } } block := body.AppendNewBlock(key, labels) if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil { return true } } // If all child values are mappings, treat each child key as a labelled instance of this block type if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled { return true } // No labels detected, render as unlabelled block block := body.AppendNewBlock(key, nil) if err := he.encodeNodeAttributes(block.Body(), valueNode); err == nil { return true } return false } // encodeNode encodes a CandidateNode directly to HCL, preserving style information func (he *hclEncoder) encodeNode(body *hclwrite.Body, node *CandidateNode) error { if node.Kind != MappingNode { return fmt.Errorf("HCL encoder expects a mapping at the root level, got %v", kindToString(node.Kind)) } for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] key := keyNode.Value // Render as block or attribute depending on value type if he.encodeBlockIfMapping(body, key, valueNode) { continue } // Render as attribute: key = value if err := he.encodeAttribute(body, key, valueNode); err != nil { return err } } return nil } // mappingChildrenAllMappings reports whether all values in a mapping node are non-flow mappings. func mappingChildrenAllMappings(node *CandidateNode) bool { if node == nil || node.Kind != MappingNode || node.Style == FlowStyle { return false } if len(node.Content) == 0 { return false } for i := 0; i < len(node.Content); i += 2 { childVal := node.Content[i+1] if childVal.Kind != MappingNode || childVal.Style == FlowStyle { return false } } return true } // encodeMappingChildrenAsBlocks emits a block for each mapping child, treating the child key as a label. // Returns handled=true when it emitted blocks. func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockType string, valueNode *CandidateNode) (bool, error) { if !mappingChildrenAllMappings(valueNode) { return false, nil } // Only emit as separate blocks if EncodeSeparate is true // This allows the encoder to respect the original block structure preserved by the decoder if !valueNode.EncodeSeparate { return false, nil } for i := 0; i < len(valueNode.Content); i += 2 { childKey := valueNode.Content[i].Value childVal := valueNode.Content[i+1] // Check if this child also represents multiple blocks (all children are mappings) if mappingChildrenAllMappings(childVal) { // Recursively emit each grandchild as a separate block with extended labels for j := 0; j < len(childVal.Content); j += 2 { grandchildKey := childVal.Content[j].Value grandchildVal := childVal.Content[j+1] labels := []string{childKey, grandchildKey} // Try to extract additional labels if this is a single-entry chain if extraLabels, bodyNode, ok := extractBlockLabels(grandchildVal); ok { labels = append(labels, extraLabels...) grandchildVal = bodyNode } block := body.AppendNewBlock(blockType, labels) if err := he.encodeNodeAttributes(block.Body(), grandchildVal); err != nil { return true, err } } } else { // Single block with this child as label(s) labels := []string{childKey} if extraLabels, bodyNode, ok := extractBlockLabels(childVal); ok { labels = append(labels, extraLabels...) childVal = bodyNode } block := body.AppendNewBlock(blockType, labels) if err := he.encodeNodeAttributes(block.Body(), childVal); err != nil { return true, err } } } return true, nil } // encodeNodeAttributes encodes the attributes of a mapping node (used for blocks) func (he *hclEncoder) encodeNodeAttributes(body *hclwrite.Body, node *CandidateNode) error { if node.Kind != MappingNode { return fmt.Errorf("expected mapping node for block body") } for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] key := keyNode.Value // Render as block or attribute depending on value type if he.encodeBlockIfMapping(body, key, valueNode) { continue } // Render attribute for non-block value if err := he.encodeAttribute(body, key, valueNode); err != nil { return err } } return nil } // extractBlockLabels detects a chain of single-entry mappings that encode block labels. // It returns the collected labels and the final mapping to be used as the block body. // Pattern: {label1: {label2: { ... {bodyMap} }}} func extractBlockLabels(node *CandidateNode) ([]string, *CandidateNode, bool) { var labels []string current := node for current != nil && current.Kind == MappingNode && len(current.Content) == 2 { keyNode := current.Content[0] valNode := current.Content[1] if valNode.Kind != MappingNode { break } labels = append(labels, keyNode.Value) // If the child is itself a single mapping entry with a mapping value, keep descending. if len(valNode.Content) == 2 && valNode.Content[1].Kind == MappingNode { current = valNode continue } // Otherwise, we have reached the body mapping. return labels, valNode, true } return nil, nil, false } // nodeToCtyValue converts a CandidateNode directly to cty.Value, preserving order func nodeToCtyValue(node *CandidateNode) (cty.Value, error) { switch node.Kind { case ScalarNode: // Parse scalar value based on its tag switch node.Tag { case "!!bool": return cty.BoolVal(node.Value == "true"), nil case "!!int": var i int64 _, err := fmt.Sscanf(node.Value, "%d", &i) if err != nil { return cty.NilVal, err } return cty.NumberIntVal(i), nil case "!!float": var f float64 _, err := fmt.Sscanf(node.Value, "%f", &f) if err != nil { return cty.NilVal, err } return cty.NumberFloatVal(f), nil case "!!null": return cty.NullVal(cty.DynamicPseudoType), nil default: // Default to string return cty.StringVal(node.Value), nil } case MappingNode: // Preserve order by iterating Content directly m := make(map[string]cty.Value) for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] v, err := nodeToCtyValue(valueNode) if err != nil { return cty.NilVal, err } m[keyNode.Value] = v } return cty.ObjectVal(m), nil case SequenceNode: vals := make([]cty.Value, len(node.Content)) for i, item := range node.Content { v, err := nodeToCtyValue(item) if err != nil { return cty.NilVal, err } vals[i] = v } return cty.TupleVal(vals), nil case AliasNode: return cty.NilVal, fmt.Errorf("HCL encoder does not support aliases") default: return cty.NilVal, fmt.Errorf("unsupported node kind: %v", node.Kind) } } ================================================ FILE: pkg/yqlib/encoder_ini.go ================================================ //go:build !yq_noini package yqlib import ( "bytes" "fmt" "io" "github.com/go-ini/ini" ) type iniEncoder struct { indentString string } // NewINIEncoder creates a new INI encoder func NewINIEncoder() Encoder { // Hardcoded indent value of 0, meaning no additional spacing. return &iniEncoder{""} } // CanHandleAliases indicates whether the encoder supports aliases. INI does not support aliases. func (ie *iniEncoder) CanHandleAliases() bool { return false } // PrintDocumentSeparator is a no-op since INI does not support multiple documents. func (ie *iniEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } // PrintLeadingContent is a no-op since INI does not support leading content or comments at the encoder level. func (ie *iniEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } // Encode converts a CandidateNode into INI format and writes it to the provided writer. func (ie *iniEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debugf("I need to encode %v", NodeToString(node)) log.Debugf("kids %v", len(node.Content)) if node.Kind == ScalarNode { return writeStringINI(writer, node.Value+"\n") } // Create a new INI configuration. cfg := ini.Empty() if node.Kind == MappingNode { // Default section for key-value pairs at the root level. defaultSection, err := cfg.NewSection(ini.DefaultSection) if err != nil { return err } // Process the node's content. for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] key := keyNode.Value switch valueNode.Kind { case ScalarNode: // Add key-value pair to the default section. _, err := defaultSection.NewKey(key, valueNode.Value) if err != nil { return err } case MappingNode: // Create a new section for nested MappingNode. section, err := cfg.NewSection(key) if err != nil { return err } // Process nested key-value pairs. for j := 0; j < len(valueNode.Content); j += 2 { nestedKeyNode := valueNode.Content[j] nestedValueNode := valueNode.Content[j+1] if nestedValueNode.Kind == ScalarNode { _, err := section.NewKey(nestedKeyNode.Value, nestedValueNode.Value) if err != nil { return err } } else { log.Debugf("Skipping nested non-scalar value for key %s: %v", nestedKeyNode.Value, nestedValueNode.Kind) } } default: log.Debugf("Skipping non-scalar value for key %s: %v", key, valueNode.Kind) } } } else { return fmt.Errorf("INI encoder supports only MappingNode at the root level, got %v", node.Kind) } // Use a buffer to store the INI output as the library doesn't support direct io.Writer with indent. var buffer bytes.Buffer _, err := cfg.WriteToIndent(&buffer, ie.indentString) if err != nil { return err } // Write the buffer content to the provided writer. _, err = writer.Write(buffer.Bytes()) return err } // writeStringINI is a helper function to write a string to the provided writer for INI encoder. func writeStringINI(writer io.Writer, content string) error { _, err := writer.Write([]byte(content)) return err } ================================================ FILE: pkg/yqlib/encoder_json.go ================================================ //go:build !yq_nojson package yqlib import ( "bytes" "io" "github.com/goccy/go-json" ) type jsonEncoder struct { prefs JsonPreferences indentString string } func NewJSONEncoder(prefs JsonPreferences) Encoder { var indentString = "" for index := 0; index < prefs.Indent; index++ { indentString = indentString + " " } return &jsonEncoder{prefs, indentString} } func (je *jsonEncoder) CanHandleAliases() bool { return false } func (je *jsonEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (je *jsonEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debugf("I need to encode %v", NodeToString(node)) log.Debugf("kids %v", len(node.Content)) if node.Kind == ScalarNode && je.prefs.UnwrapScalar { return writeString(writer, node.Value+"\n") } destination := writer tempBuffer := bytes.NewBuffer(nil) if je.prefs.ColorsEnabled { destination = tempBuffer } var encoder = json.NewEncoder(destination) encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, > encoder.SetIndent("", je.indentString) err := encoder.Encode(node) if err != nil { return err } if je.prefs.ColorsEnabled { return colorizeAndPrint(tempBuffer.Bytes(), writer) } return nil } ================================================ FILE: pkg/yqlib/encoder_kyaml.go ================================================ //go:build !yq_nokyaml package yqlib import ( "bytes" "io" "regexp" "strconv" "strings" ) type kyamlEncoder struct { prefs KYamlPreferences } func NewKYamlEncoder(prefs KYamlPreferences) Encoder { return &kyamlEncoder{prefs: prefs} } func (ke *kyamlEncoder) CanHandleAliases() bool { // KYAML is a restricted subset; avoid emitting anchors/aliases. return false } func (ke *kyamlEncoder) PrintDocumentSeparator(writer io.Writer) error { return PrintYAMLDocumentSeparator(writer, ke.prefs.PrintDocSeparators) } func (ke *kyamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { return PrintYAMLLeadingContent(writer, content, ke.prefs.PrintDocSeparators, ke.prefs.ColorsEnabled) } func (ke *kyamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debug("encoderKYaml - going to print %v", NodeToString(node)) if node.Kind == ScalarNode && ke.prefs.UnwrapScalar { return writeString(writer, node.Value+"\n") } destination := writer tempBuffer := bytes.NewBuffer(nil) if ke.prefs.ColorsEnabled { destination = tempBuffer } // Mirror the YAML encoder behaviour: trailing comments on the document root // are stored in FootComment and need to be printed after the document. trailingContent := node.FootComment if err := ke.writeCommentBlock(destination, node.HeadComment, 0); err != nil { return err } if err := ke.writeNode(destination, node, 0); err != nil { return err } if err := ke.writeInlineComment(destination, node.LineComment); err != nil { return err } if err := writeString(destination, "\n"); err != nil { return err } if err := ke.PrintLeadingContent(destination, trailingContent); err != nil { return err } if ke.prefs.ColorsEnabled { return colorizeAndPrint(tempBuffer.Bytes(), writer) } return nil } func (ke *kyamlEncoder) writeNode(writer io.Writer, node *CandidateNode, indent int) error { switch node.Kind { case MappingNode: return ke.writeMapping(writer, node, indent) case SequenceNode: return ke.writeSequence(writer, node, indent) case ScalarNode: return writeString(writer, ke.formatScalar(node)) case AliasNode: // Should have been exploded by the printer, but handle defensively. if node.Alias == nil { return writeString(writer, "null") } return ke.writeNode(writer, node.Alias, indent) default: return writeString(writer, "null") } } func (ke *kyamlEncoder) writeMapping(writer io.Writer, node *CandidateNode, indent int) error { if len(node.Content) == 0 { return writeString(writer, "{}") } if err := writeString(writer, "{\n"); err != nil { return err } for i := 0; i+1 < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] entryIndent := indent + ke.prefs.Indent if err := ke.writeCommentBlock(writer, keyNode.HeadComment, entryIndent); err != nil { return err } if valueNode.HeadComment != "" && valueNode.HeadComment != keyNode.HeadComment { if err := ke.writeCommentBlock(writer, valueNode.HeadComment, entryIndent); err != nil { return err } } if err := ke.writeIndent(writer, entryIndent); err != nil { return err } if err := writeString(writer, ke.formatKey(keyNode)); err != nil { return err } if err := writeString(writer, ": "); err != nil { return err } if err := ke.writeNode(writer, valueNode, entryIndent); err != nil { return err } // Always emit a trailing comma; KYAML encourages explicit separators, // and this ensures all quoted strings have a trailing `",` as requested. if err := writeString(writer, ","); err != nil { return err } inline := valueNode.LineComment if inline == "" { inline = keyNode.LineComment } if err := ke.writeInlineComment(writer, inline); err != nil { return err } if err := writeString(writer, "\n"); err != nil { return err } foot := valueNode.FootComment if foot == "" { foot = keyNode.FootComment } if err := ke.writeCommentBlock(writer, foot, entryIndent); err != nil { return err } } if err := ke.writeIndent(writer, indent); err != nil { return err } return writeString(writer, "}") } func (ke *kyamlEncoder) writeSequence(writer io.Writer, node *CandidateNode, indent int) error { if len(node.Content) == 0 { return writeString(writer, "[]") } if err := writeString(writer, "[\n"); err != nil { return err } for _, child := range node.Content { itemIndent := indent + ke.prefs.Indent if err := ke.writeCommentBlock(writer, child.HeadComment, itemIndent); err != nil { return err } if err := ke.writeIndent(writer, itemIndent); err != nil { return err } if err := ke.writeNode(writer, child, itemIndent); err != nil { return err } if err := writeString(writer, ","); err != nil { return err } if err := ke.writeInlineComment(writer, child.LineComment); err != nil { return err } if err := writeString(writer, "\n"); err != nil { return err } if err := ke.writeCommentBlock(writer, child.FootComment, itemIndent); err != nil { return err } } if err := ke.writeIndent(writer, indent); err != nil { return err } return writeString(writer, "]") } func (ke *kyamlEncoder) writeIndent(writer io.Writer, indent int) error { if indent <= 0 { return nil } return writeString(writer, strings.Repeat(" ", indent)) } func (ke *kyamlEncoder) formatKey(keyNode *CandidateNode) string { // KYAML examples use bare keys. Quote keys only when needed. key := keyNode.Value if isValidKYamlBareKey(key) { return key } return `"` + escapeDoubleQuotedString(key) + `"` } func (ke *kyamlEncoder) formatScalar(node *CandidateNode) string { switch node.Tag { case "!!null": return "null" case "!!bool": return strings.ToLower(node.Value) case "!!int", "!!float": return node.Value case "!!str": return `"` + escapeDoubleQuotedString(node.Value) + `"` default: // Fall back to a string representation to avoid implicit typing surprises. return `"` + escapeDoubleQuotedString(node.Value) + `"` } } var kyamlBareKeyRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_-]*$`) func isValidKYamlBareKey(s string) bool { // Conservative: require an identifier-like key; otherwise quote. if s == "" { return false } return kyamlBareKeyRe.MatchString(s) } func escapeDoubleQuotedString(s string) string { var b strings.Builder b.Grow(len(s) + 2) for _, r := range s { switch r { case '\\': b.WriteString(`\\`) case '"': b.WriteString(`\"`) case '\n': b.WriteString(`\n`) case '\r': b.WriteString(`\r`) case '\t': b.WriteString(`\t`) default: if r < 0x20 { // YAML double-quoted strings support \uXXXX escapes. b.WriteString(`\u`) hex := "0000" + strings.ToUpper(strconv.FormatInt(int64(r), 16)) b.WriteString(hex[len(hex)-4:]) } else { b.WriteRune(r) } } } return b.String() } func (ke *kyamlEncoder) writeCommentBlock(writer io.Writer, comment string, indent int) error { if strings.TrimSpace(comment) == "" { return nil } lines := strings.Split(strings.ReplaceAll(comment, "\r\n", "\n"), "\n") for _, line := range lines { trimmed := strings.TrimSpace(line) if trimmed == "" { continue } if err := ke.writeIndent(writer, indent); err != nil { return err } toWrite := line if !commentLineRe.MatchString(toWrite) { toWrite = "# " + toWrite } if err := writeString(writer, toWrite); err != nil { return err } if err := writeString(writer, "\n"); err != nil { return err } } return nil } func (ke *kyamlEncoder) writeInlineComment(writer io.Writer, comment string) error { comment = strings.TrimSpace(strings.ReplaceAll(comment, "\r\n", "\n")) if comment == "" { return nil } lines := strings.Split(comment, "\n") first := strings.TrimSpace(lines[0]) if first == "" { return nil } if !strings.HasPrefix(first, "#") { first = "# " + first } if err := writeString(writer, " "); err != nil { return err } return writeString(writer, first) } ================================================ FILE: pkg/yqlib/encoder_lua.go ================================================ //go:build !yq_nolua package yqlib import ( "fmt" "io" "strings" ) type luaEncoder struct { docPrefix string docSuffix string indent int indentStr string unquoted bool globals bool escape *strings.Replacer } func (le *luaEncoder) CanHandleAliases() bool { return false } func NewLuaEncoder(prefs LuaPreferences) Encoder { escape := strings.NewReplacer( "\000", "\\000", "\001", "\\001", "\002", "\\002", "\003", "\\003", "\004", "\\004", "\005", "\\005", "\006", "\\006", "\007", "\\a", "\010", "\\b", "\011", "\\t", "\012", "\\n", "\013", "\\v", "\014", "\\f", "\015", "\\r", "\016", "\\014", "\017", "\\015", "\020", "\\016", "\021", "\\017", "\022", "\\018", "\023", "\\019", "\024", "\\020", "\025", "\\021", "\026", "\\022", "\027", "\\023", "\030", "\\024", "\031", "\\025", "\032", "\\026", "\033", "\\027", "\034", "\\028", "\035", "\\029", "\036", "\\030", "\037", "\\031", "\"", "\\\"", "'", "\\'", "\\", "\\\\", "\177", "\\127", ) unescape := strings.NewReplacer( "\\'", "'", "\\\"", "\"", "\\n", "\n", "\\r", "\r", "\\t", "\t", "\\\\", "\\", ) return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape} } func (le *luaEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (le *luaEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error { quote := "\"" switch node.Style { case LiteralStyle, FoldedStyle, FlowStyle: for i := 0; i < 10; i++ { if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") { err := writeString(writer, "["+strings.Repeat("=", i)+"[\n") if err != nil { return err } err = writeString(writer, node.Value) if err != nil { return err } return writeString(writer, "]"+strings.Repeat("=", i)+"]") } } case SingleQuotedStyle: quote = "'" // fallthrough to regular ol' string } return writeString(writer, quote+le.escape.Replace(node.Value)+quote) } func (le *luaEncoder) writeIndent(writer io.Writer) error { if le.indentStr == "" { return nil } err := writeString(writer, "\n") if err != nil { return err } return writeString(writer, strings.Repeat(le.indentStr, le.indent)) } func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error { err := writeString(writer, "{") if err != nil { return err } le.indent++ for _, child := range node.Content { err = le.writeIndent(writer) if err != nil { return err } err := le.encodeAny(writer, child) if err != nil { return err } err = writeString(writer, ",") if err != nil { return err } if child.LineComment != "" { sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") err = writeString(writer, " --"+sansPrefix) if err != nil { return err } } } le.indent-- if len(node.Content) != 0 { err = le.writeIndent(writer) if err != nil { return err } } return writeString(writer, "}") } func needsQuoting(s string) bool { // known keywords as of Lua 5.4 switch s { case "do", "and", "else", "break", "if", "end", "goto", "false", "in", "for", "then", "local", "or", "nil", "true", "until", "elseif", "function", "not", "repeat", "return", "while": return true } // [%a_][%w_]* for i, c := range s { if i == 0 { // keeping for legacy reasons, upgraded linter //nolint:staticcheck if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { return true } } else { // keeping for legacy reasons, upgraded linter //nolint:staticcheck if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { return true } } } return false } func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error { if !global { err := writeString(writer, "{") if err != nil { return err } le.indent++ } for i, child := range node.Content { if (i % 2) == 1 { // value err := le.encodeAny(writer, child) if err != nil { return err } err = writeString(writer, ";") if err != nil { return err } } else { // key if !global || i > 0 { err := le.writeIndent(writer) if err != nil { return err } } if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) { err := writeString(writer, child.Value+" = ") if err != nil { return err } } else { if global { // This only works in Lua 5.2+ err := writeString(writer, "_ENV") if err != nil { return err } } err := writeString(writer, "[") if err != nil { return err } err = le.encodeAny(writer, child) if err != nil { return err } err = writeString(writer, "] = ") if err != nil { return err } } } if child.LineComment != "" { sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) if err != nil { return err } if (i % 2) == 0 { // newline and indent after comments on keys err = le.writeIndent(writer) if err != nil { return err } } } } if global { return writeString(writer, "\n") } le.indent-- if len(node.Content) != 0 { err := le.writeIndent(writer) if err != nil { return err } } return writeString(writer, "}") } func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error { switch node.Kind { case SequenceNode: return le.encodeArray(writer, node) case MappingNode: return le.encodeMap(writer, node, false) case ScalarNode: switch node.Tag { case "!!str": return le.encodeString(writer, node) case "!!null": // TODO reject invalid use as a table key return writeString(writer, "nil") case "!!bool": // Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has // lower case return writeString(writer, strings.ToLower(node.Value)) case "!!int": if strings.HasPrefix(node.Value, "0o") { _, octalValue, err := parseInt64(node.Value) if err != nil { return err } return writeString(writer, fmt.Sprintf("%d", octalValue)) } return writeString(writer, strings.ToLower(node.Value)) case "!!float": switch strings.ToLower(node.Value) { case ".inf", "+.inf": return writeString(writer, "(1/0)") case "-.inf": return writeString(writer, "(-1/0)") case ".nan": return writeString(writer, "(0/0)") default: return writeString(writer, node.Value) } default: return fmt.Errorf("lua encoder NYI -- %s", node.Tag) } default: return fmt.Errorf("lua encoder NYI -- %s", node.Tag) } } func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error { err := writeString(writer, le.docPrefix) if err != nil { return err } err = le.encodeAny(writer, node) if err != nil { return err } return writeString(writer, le.docSuffix) } func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error { if le.globals { if node.Kind != MappingNode { return fmt.Errorf("--lua-global requires a top level MappingNode") } return le.encodeMap(writer, node, true) } return le.encodeTopLevel(writer, node) } ================================================ FILE: pkg/yqlib/encoder_properties.go ================================================ //go:build !yq_noprops package yqlib import ( "bufio" "errors" "fmt" "io" "strings" "github.com/magiconair/properties" ) type propertiesEncoder struct { prefs PropertiesPreferences } func NewPropertiesEncoder(prefs PropertiesPreferences) Encoder { return &propertiesEncoder{ prefs: prefs, } } func (pe *propertiesEncoder) CanHandleAliases() bool { return false } func (pe *propertiesEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content string) error { reader := bufio.NewReader(strings.NewReader(content)) for { readline, errReading := reader.ReadString('\n') if errReading != nil && !errors.Is(errReading, io.EOF) { return errReading } if strings.Contains(readline, "$yqDocSeparator$") { if err := pe.PrintDocumentSeparator(writer); err != nil { return err } } else { if err := writeString(writer, readline); err != nil { return err } } if errors.Is(errReading, io.EOF) { if readline != "" { // the last comment we read didn't have a newline, put one in if err := writeString(writer, "\n"); err != nil { return err } } break } } return nil } func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.Kind == ScalarNode { return writeString(writer, node.Value+"\n") } mapKeysToStrings(node) p := properties.NewProperties() p.WriteSeparator = pe.prefs.KeyValueSeparator err := pe.doEncode(p, node, "", nil) if err != nil { return err } _, err = p.WriteComment(writer, "#", properties.UTF8) return err } func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *CandidateNode, path string, keyNode *CandidateNode) error { comments := "" if keyNode != nil { // include the key node comments if present comments = headAndLineComment(keyNode) } comments = comments + headAndLineComment(node) commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ") p.SetComments(path, strings.Split(commentsWithSpaces, "\n")) switch node.Kind { case ScalarNode: var nodeValue string if pe.prefs.UnwrapScalar || !strings.Contains(node.Value, " ") { nodeValue = node.Value } else { nodeValue = fmt.Sprintf("%q", node.Value) } _, _, err := p.Set(path, nodeValue) return err case SequenceNode: return pe.encodeArray(p, node.Content, path) case MappingNode: return pe.encodeMap(p, node.Content, path) case AliasNode: return pe.doEncode(p, node.Alias, path, nil) default: return fmt.Errorf("unsupported node %v", node.Tag) } } func (pe *propertiesEncoder) appendPath(path string, key interface{}) string { if path == "" { return fmt.Sprintf("%v", key) } switch key.(type) { case int: if pe.prefs.UseArrayBrackets { return fmt.Sprintf("%v[%v]", path, key) } } return fmt.Sprintf("%v.%v", path, key) } func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*CandidateNode, path string) error { for index, child := range kids { err := pe.doEncode(p, child, pe.appendPath(path, index), nil) if err != nil { return err } } return nil } func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*CandidateNode, path string) error { for index := 0; index < len(kids); index = index + 2 { key := kids[index] value := kids[index+1] err := pe.doEncode(p, value, pe.appendPath(path, key.Value), key) if err != nil { return err } } return nil } ================================================ FILE: pkg/yqlib/encoder_properties_test.go ================================================ package yqlib import ( "bufio" "bytes" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) type keyValuePair struct { key string value string comment string } func (kv *keyValuePair) String(unwrap bool, sep string) string { builder := strings.Builder{} if kv.comment != "" { builder.WriteString(kv.comment) builder.WriteString("\n") } builder.WriteString(kv.key) builder.WriteString(sep) if unwrap { builder.WriteString(kv.value) } else { builder.WriteString("\"") builder.WriteString(kv.value) builder.WriteString("\"") } return builder.String() } type testProperties struct { pairs []keyValuePair } func (tp *testProperties) String(unwrap bool, sep string) string { kvs := []string{} for _, kv := range tp.pairs { kvs = append(kvs, kv.String(unwrap, sep)) } return strings.Join(kvs, "\n") } func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string { var output bytes.Buffer writer := bufio.NewWriter(&output) var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar}) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode) err = propsEncoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() return strings.TrimSuffix(output.String(), "\n") } func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) { wraps := []bool{} if testUnwrapped { wraps = append(wraps, true) } if testWrapped { wraps = append(wraps, false) } for _, unwrap := range wraps { for _, sep := range []string{" = ", ";", "=", " "} { var actualProps = yamlToProps(sampleYaml, unwrap, sep) test.AssertResult(t, props.String(unwrap, sep), actualProps) } } } func TestPropertiesEncoderSimple(t *testing.T) { var sampleYaml = `a: 'bob cool'` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a", value: "bob cool", }, }, }, true, true, ) } func TestPropertiesEncoderSimpleWithComments(t *testing.T) { var sampleYaml = `a: 'bob cool' # line` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a", value: "bob cool", comment: "# line", }, }, }, true, true, ) } func TestPropertiesEncoderDeep(t *testing.T) { var sampleYaml = `a: b: "bob cool" ` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a.b", value: "bob cool", }, }, }, true, true, ) } func TestPropertiesEncoderDeepWithComments(t *testing.T) { var sampleYaml = `a: # a thing b: "bob cool" # b thing ` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a.b", value: "bob cool", comment: "# b thing", }, }, }, true, true, ) } func TestPropertiesEncoderArray_Unwrapped(t *testing.T) { var sampleYaml = `a: b: [{c: dog}, {c: cat}] ` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a.b.0.c", value: "dog", }, { key: "a.b.1.c", value: "cat", }, }, }, true, false, ) } func TestPropertiesEncoderArray_Wrapped(t *testing.T) { var sampleYaml = `a: b: [{c: dog named jim}, {c: cat named jim}] ` doTest( t, sampleYaml, testProperties{ pairs: []keyValuePair{ { key: "a.b.0.c", value: "dog named jim", }, { key: "a.b.1.c", value: "cat named jim", }, }, }, false, true, ) } ================================================ FILE: pkg/yqlib/encoder_sh.go ================================================ //go:build !yq_nosh package yqlib import ( "fmt" "io" "regexp" "strings" ) var unsafeChars = regexp.MustCompile(`[^\w@%+=:,./-]`) type shEncoder struct { quoteAll bool } func NewShEncoder() Encoder { return &shEncoder{false} } func (e *shEncoder) CanHandleAliases() bool { return false } func (e *shEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.guessTagFromCustomType() != "!!str" { return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) } return writeString(writer, e.encode(node.Value)) } // put any (shell-unsafe) characters into a single-quoted block, close the block lazily func (e *shEncoder) encode(input string) string { const quote = '\'' var inQuoteBlock = false var encoded strings.Builder encoded.Grow(len(input)) for _, ir := range input { // open or close a single-quote block if ir == quote { if inQuoteBlock { // get out of a quote block for an input quote encoded.WriteRune(quote) inQuoteBlock = !inQuoteBlock } // escape the quote with a backslash encoded.WriteRune('\\') } else { if e.shouldQuote(ir) && !inQuoteBlock { // start a quote block for any (unsafe) characters encoded.WriteRune(quote) inQuoteBlock = !inQuoteBlock } } // pass on the input character encoded.WriteRune(ir) } // close any pending quote block if inQuoteBlock { encoded.WriteRune(quote) } return encoded.String() } func (e *shEncoder) shouldQuote(ir rune) bool { return e.quoteAll || unsafeChars.MatchString(string(ir)) } ================================================ FILE: pkg/yqlib/encoder_shellvariables.go ================================================ //go:build !yq_noshell package yqlib import ( "fmt" "io" "strings" "unicode/utf8" "golang.org/x/text/unicode/norm" ) type shellVariablesEncoder struct { prefs ShellVariablesPreferences } func NewShellVariablesEncoder() Encoder { return &shellVariablesEncoder{ prefs: ConfiguredShellVariablesPreferences, } } func (pe *shellVariablesEncoder) CanHandleAliases() bool { return false } func (pe *shellVariablesEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error { mapKeysToStrings(node) err := pe.doEncode(&writer, node, "") if err != nil { return err } return err } func (pe *shellVariablesEncoder) doEncode(w *io.Writer, node *CandidateNode, path string) error { // Note this drops all comments. switch node.Kind { case ScalarNode: nonemptyPath := path if path == "" { // We can't assign an empty variable "=somevalue" because that would error out if sourced in a shell, // nor can we use "_" as a variable name ($_ is a special shell variable that can't be assigned)... // let's just pick a fallback key to use if we are encoding a single scalar nonemptyPath = "value" } var valueString string if pe.prefs.UnwrapScalar { valueString = node.Value } else { valueString = quoteValue(node.Value) } _, err := io.WriteString(*w, nonemptyPath+"="+valueString+"\n") return err case SequenceNode: for index, child := range node.Content { err := pe.doEncode(w, child, pe.appendPath(path, index)) if err != nil { return err } } return nil case MappingNode: for index := 0; index < len(node.Content); index = index + 2 { key := node.Content[index] value := node.Content[index+1] err := pe.doEncode(w, value, pe.appendPath(path, key.Value)) if err != nil { return err } } return nil case AliasNode: return pe.doEncode(w, node.Alias, path) default: return fmt.Errorf("unsupported node %v", node.Tag) } } func (pe *shellVariablesEncoder) appendPath(cookedPath string, rawKey interface{}) string { // Shell variable names must match // [a-zA-Z_]+[a-zA-Z0-9_]* // // While this is not mandated by POSIX, which is quite lenient, it is // what shells (for example busybox ash *) allow in practice. // // Since yaml names can contain basically any character, we will process them according to these steps: // // 1. apply unicode compatibility decomposition NFKD (this will convert accented // letters to letters followed by accents, split ligatures, replace exponents // with the corresponding digit, etc. // // 2. discard non-ASCII characters as well as ASCII control characters (ie. anything // with code point < 32 or > 126), this will eg. discard accents but keep the base // unaccented letter because of NFKD above // // 3. replace all non-alphanumeric characters with _ // // Moreover, for the root key only, we will prepend an underscore if what results from the steps above // does not start with [a-zA-Z_] (ie. if the root key starts with a digit). // // Note this is NOT a 1:1 mapping. // // (*) see endofname.c from https://git.busybox.net/busybox/tag/?h=1_36_0 // XXX empty strings key := strings.Map(func(r rune) rune { if isAlphaNumericOrUnderscore(r) { return r } else if r < 32 || 126 < r { return -1 } return '_' }, norm.NFKD.String(fmt.Sprintf("%v", rawKey))) if cookedPath == "" { firstRune, _ := utf8.DecodeRuneInString(key) if !isAlphaOrUnderscore(firstRune) { return "_" + key } return key } return cookedPath + pe.prefs.KeySeparator + key } func quoteValue(value string) string { needsQuoting := false for _, r := range value { if !isAlphaNumericOrUnderscore(r) { needsQuoting = true break } } if needsQuoting { return "'" + strings.ReplaceAll(value, "'", "'\"'\"'") + "'" } return value } func isAlphaOrUnderscore(r rune) bool { return ('a' <= r && r <= 'z') || ('A' <= r && r <= 'Z') || r == '_' } func isAlphaNumericOrUnderscore(r rune) bool { return isAlphaOrUnderscore(r) || ('0' <= r && r <= '9') } ================================================ FILE: pkg/yqlib/encoder_shellvariables_test.go ================================================ package yqlib import ( "bufio" "bytes" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) func assertEncodesTo(t *testing.T, yaml string, shellvars string) { var output bytes.Buffer writer := bufio.NewWriter(&output) var encoder = NewShellVariablesEncoder() inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode) err = encoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n")) } func TestShellVariablesEncoderNonquoting(t *testing.T) { assertEncodesTo(t, "a: alice", "a=alice") } func TestShellVariablesEncoderQuoting(t *testing.T) { assertEncodesTo(t, "a: Lewis Carroll", "a='Lewis Carroll'") } func TestShellVariablesEncoderQuotesQuoting(t *testing.T) { assertEncodesTo(t, "a: Lewis Carroll's Alice", "a='Lewis Carroll'\"'\"'s Alice'") } func TestShellVariablesEncoderStripComments(t *testing.T) { assertEncodesTo(t, "a: Alice # comment", "a=Alice") } func TestShellVariablesEncoderMap(t *testing.T) { assertEncodesTo(t, "a:\n b: Lewis\n c: Carroll", "a_b=Lewis\na_c=Carroll") } func TestShellVariablesEncoderArray_Unwrapped(t *testing.T) { assertEncodesTo(t, "a: [{n: Alice}, {n: Bob}]", "a_0_n=Alice\na_1_n=Bob") } func TestShellVariablesEncoderKeyNonPrintable(t *testing.T) { assertEncodesTo(t, `"be\all": ring!`, "bell='ring!'") } func TestShellVariablesEncoderKeyPrintableNonAlphaNumeric(t *testing.T) { assertEncodesTo(t, `"b-e l=l": ring!`, "b_e_l_l='ring!'") } func TestShellVariablesEncoderKeyPrintableNonAscii(t *testing.T) { assertEncodesTo(t, `"b\u00e9ll": ring!`, "bell='ring!'") } func TestShellVariablesEncoderRootKeyStartingWithDigit(t *testing.T) { assertEncodesTo(t, "1a: onea", "_1a=onea") } func TestShellVariablesEncoderRootKeyStartingWithUnderscore(t *testing.T) { assertEncodesTo(t, "_key: value", "_key=value") } func TestShellVariablesEncoderChildStartingWithUnderscore(t *testing.T) { assertEncodesTo(t, "root:\n _child: value", "root__child=value") } func TestShellVariablesEncoderEmptyValue(t *testing.T) { assertEncodesTo(t, "empty:", "empty=") } func TestShellVariablesEncoderEmptyArray(t *testing.T) { assertEncodesTo(t, "empty: []", "") } func TestShellVariablesEncoderEmptyMap(t *testing.T) { assertEncodesTo(t, "empty: {}", "") } func TestShellVariablesEncoderScalarNode(t *testing.T) { assertEncodesTo(t, "some string", "value='some string'") } func assertEncodesToWithSeparator(t *testing.T, yaml string, shellvars string, separator string) { var output bytes.Buffer writer := bufio.NewWriter(&output) // Save the original separator originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator defer func() { ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator }() // Set the custom separator ConfiguredShellVariablesPreferences.KeySeparator = separator var encoder = NewShellVariablesEncoder() inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode) err = encoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n")) } func TestShellVariablesEncoderCustomSeparator(t *testing.T) { assertEncodesToWithSeparator(t, "a:\n b: Lewis\n c: Carroll", "a__b=Lewis\na__c=Carroll", "__") } func TestShellVariablesEncoderCustomSeparatorNested(t *testing.T) { assertEncodesToWithSeparator(t, "my_app:\n db_config:\n host: localhost", "my_app__db_config__host=localhost", "__") } func TestShellVariablesEncoderCustomSeparatorArray(t *testing.T) { assertEncodesToWithSeparator(t, "a: [{n: Alice}, {n: Bob}]", "a__0__n=Alice\na__1__n=Bob", "__") } func TestShellVariablesEncoderCustomSeparatorSingleChar(t *testing.T) { assertEncodesToWithSeparator(t, "a:\n b: value", "aXb=value", "X") } func assertEncodesToUnwrapped(t *testing.T, yaml string, shellvars string) { var output bytes.Buffer writer := bufio.NewWriter(&output) originalUnwrapScalar := ConfiguredShellVariablesPreferences.UnwrapScalar defer func() { ConfiguredShellVariablesPreferences.UnwrapScalar = originalUnwrapScalar }() ConfiguredShellVariablesPreferences.UnwrapScalar = true var encoder = NewShellVariablesEncoder() inputs, err := readDocuments(strings.NewReader(yaml), "test.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode) err = encoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, shellvars, strings.TrimSuffix(output.String(), "\n")) } func TestShellVariablesEncoderUnwrapScalar(t *testing.T) { assertEncodesToUnwrapped(t, "a: Lewis Carroll", "a=Lewis Carroll") assertEncodesToUnwrapped(t, "b: 123", "b=123") assertEncodesToUnwrapped(t, "c: true", "c=true") assertEncodesToUnwrapped(t, "d: value with spaces", "d=value with spaces") } ================================================ FILE: pkg/yqlib/encoder_test.go ================================================ //go:build !yq_nojson package yqlib import ( "bufio" "bytes" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) func yamlToJSON(t *testing.T, sampleYaml string, indent int) string { t.Helper() var output bytes.Buffer writer := bufio.NewWriter(&output) prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = indent prefs.UnwrapScalar = false var jsonEncoder = NewJSONEncoder(prefs) inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } node := inputs.Front().Value.(*CandidateNode) log.Debugf("%v", NodeToString(node)) // log.Debugf("Content[0] %v", NodeToString(node.Content[0])) err = jsonEncoder.Encode(writer, node) if err != nil { panic(err) } writer.Flush() return strings.TrimSuffix(output.String(), "\n") } func TestJSONEncoderPreservesObjectOrder(t *testing.T) { var sampleYaml = `zabbix: winner apple: great banana: - {cobra: kai, angus: bob} ` var expectedJSON = `{ "zabbix": "winner", "apple": "great", "banana": [ { "cobra": "kai", "angus": "bob" } ] }` var actualJSON = yamlToJSON(t, sampleYaml, 2) test.AssertResult(t, expectedJSON, actualJSON) } func TestJsonNullInArray(t *testing.T) { var sampleYaml = `[null]` var actualJSON = yamlToJSON(t, sampleYaml, 0) test.AssertResult(t, sampleYaml, actualJSON) } func TestJsonNull(t *testing.T) { var sampleYaml = `null` var actualJSON = yamlToJSON(t, sampleYaml, 0) test.AssertResult(t, sampleYaml, actualJSON) } func TestJsonNullInObject(t *testing.T) { var sampleYaml = `{x: null}` var actualJSON = yamlToJSON(t, sampleYaml, 0) test.AssertResult(t, `{"x":null}`, actualJSON) } func TestJsonEncoderDoesNotEscapeHTMLChars(t *testing.T) { var sampleYaml = `build: "( ./lint && ./format && ./compile ) < src.code"` var expectedJSON = `{"build":"( ./lint && ./format && ./compile ) < src.code"}` var actualJSON = yamlToJSON(t, sampleYaml, 0) test.AssertResult(t, expectedJSON, actualJSON) } ================================================ FILE: pkg/yqlib/encoder_toml.go ================================================ //go:build !yq_notoml package yqlib import ( "bytes" "fmt" "io" "strings" "github.com/fatih/color" ) type tomlEncoder struct { wroteRootAttr bool // Track if we wrote root-level attributes before tables prefs TomlPreferences } func NewTomlEncoder() Encoder { return NewTomlEncoderWithPrefs(ConfiguredTomlPreferences) } func NewTomlEncoderWithPrefs(prefs TomlPreferences) Encoder { return &tomlEncoder{prefs: prefs} } func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.Kind != MappingNode { // For standalone selections, TOML tests expect raw value for scalars if node.Kind == ScalarNode { return writeString(writer, node.Value+"\n") } return fmt.Errorf("TOML encoder expects a mapping at the root level") } // Encode to a buffer first if colors are enabled var buf bytes.Buffer var targetWriter io.Writer targetWriter = writer if te.prefs.ColorsEnabled { targetWriter = &buf } // Encode a root mapping as a sequence of attributes, tables, and arrays of tables if err := te.encodeRootMapping(targetWriter, node); err != nil { return err } if te.prefs.ColorsEnabled { colourised := te.colorizeToml(buf.Bytes()) _, err := writer.Write(colourised) return err } return nil } func (te *tomlEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (te *tomlEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (te *tomlEncoder) CanHandleAliases() bool { return false } // ---- helpers ---- func (te *tomlEncoder) writeComment(w io.Writer, comment string) error { if comment == "" { return nil } lines := strings.Split(comment, "\n") for _, line := range lines { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "#") { line = "# " + line } if _, err := w.Write([]byte(line + "\n")); err != nil { return err } } return nil } func (te *tomlEncoder) formatScalar(node *CandidateNode) string { switch node.Tag { case "!!str": // Quote strings per TOML spec return fmt.Sprintf("%q", node.Value) case "!!bool", "!!int", "!!float": return node.Value case "!!null": // TOML does not have null; encode as empty string return `""` default: return node.Value } } func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error { te.wroteRootAttr = false // Reset state // Write root head comment if present (at the very beginning, no leading blank line) if node.HeadComment != "" { if err := te.writeComment(w, node.HeadComment); err != nil { return err } } // Preserve existing order by iterating Content for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valNode := node.Content[i+1] if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil { return err } } return nil } // encodeTopLevelEntry encodes a key/value at the root, dispatching to attribute, table, or array-of-tables func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *CandidateNode) error { if len(path) == 0 { return fmt.Errorf("cannot encode TOML entry with empty path") } switch node.Kind { case ScalarNode: // key = value return te.writeAttribute(w, path[len(path)-1], node) case SequenceNode: // Empty arrays should be encoded as [] attributes if len(node.Content) == 0 { return te.writeArrayAttribute(w, path[len(path)-1], node) } // If all items are mappings => array of tables; else => array attribute allMaps := true for _, it := range node.Content { if it.Kind != MappingNode { allMaps = false break } } if allMaps { key := path[len(path)-1] for _, it := range node.Content { // [[key]] then body if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil { return err } } return nil } // Regular array attribute return te.writeArrayAttribute(w, path[len(path)-1], node) case MappingNode: // Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path if !node.EncodeSeparate { // If children contain mappings or arrays of mappings, prefer separate sections if te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) { return te.encodeSeparateMapping(w, path, node) } return te.writeInlineTableAttribute(w, path[len(path)-1], node) } return te.encodeSeparateMapping(w, path, node) default: return fmt.Errorf("unsupported node kind for TOML: %v", node.Kind) } } func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateNode) error { te.wroteRootAttr = true // Mark that we wrote a root attribute // Write head comment before the attribute if err := te.writeComment(w, value.HeadComment); err != nil { return err } // Write the attribute line := key + " = " + te.formatScalar(value) // Add line comment if present if value.LineComment != "" { lineComment := strings.TrimSpace(value.LineComment) if !strings.HasPrefix(lineComment, "#") { lineComment = "# " + lineComment } line += " " + lineComment } _, err := w.Write([]byte(line + "\n")) return err } func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *CandidateNode) error { te.wroteRootAttr = true // Mark that we wrote a root attribute // Write head comment before the array if err := te.writeComment(w, seq.HeadComment); err != nil { return err } // Handle empty arrays if len(seq.Content) == 0 { line := key + " = []" if seq.LineComment != "" { lineComment := strings.TrimSpace(seq.LineComment) if !strings.HasPrefix(lineComment, "#") { lineComment = "# " + lineComment } line += " " + lineComment } _, err := w.Write([]byte(line + "\n")) return err } // Check if any array elements have head comments - if so, use multiline format hasElementComments := false for _, it := range seq.Content { if it.HeadComment != "" { hasElementComments = true break } } if hasElementComments { // Write multiline array format with comments if _, err := w.Write([]byte(key + " = [\n")); err != nil { return err } for i, it := range seq.Content { // Write head comment for this element if it.HeadComment != "" { commentLines := strings.Split(it.HeadComment, "\n") for _, commentLine := range commentLines { if strings.TrimSpace(commentLine) != "" { if !strings.HasPrefix(strings.TrimSpace(commentLine), "#") { commentLine = "# " + commentLine } if _, err := w.Write([]byte(" " + commentLine + "\n")); err != nil { return err } } } } // Write the element value var itemStr string switch it.Kind { case ScalarNode: itemStr = te.formatScalar(it) case SequenceNode: nested, err := te.sequenceToInlineArray(it) if err != nil { return err } itemStr = nested case MappingNode: inline, err := te.mappingToInlineTable(it) if err != nil { return err } itemStr = inline case AliasNode: return fmt.Errorf("aliases are not supported in TOML") default: return fmt.Errorf("unsupported array item kind: %v", it.Kind) } // Always add trailing comma in multiline arrays itemStr += "," if _, err := w.Write([]byte(" " + itemStr + "\n")); err != nil { return err } // Add blank line between elements (except after the last one) if i < len(seq.Content)-1 { if _, err := w.Write([]byte("\n")); err != nil { return err } } } if _, err := w.Write([]byte("]\n")); err != nil { return err } return nil } // Join scalars or nested arrays recursively into TOML array syntax items := make([]string, 0, len(seq.Content)) for _, it := range seq.Content { switch it.Kind { case ScalarNode: items = append(items, te.formatScalar(it)) case SequenceNode: // Nested arrays: encode inline nested, err := te.sequenceToInlineArray(it) if err != nil { return err } items = append(items, nested) case MappingNode: // Inline table inside array inline, err := te.mappingToInlineTable(it) if err != nil { return err } items = append(items, inline) case AliasNode: return fmt.Errorf("aliases are not supported in TOML") default: return fmt.Errorf("unsupported array item kind: %v", it.Kind) } } line := key + " = [" + strings.Join(items, ", ") + "]" // Add line comment if present if seq.LineComment != "" { lineComment := strings.TrimSpace(seq.LineComment) if !strings.HasPrefix(lineComment, "#") { lineComment = "# " + lineComment } line += " " + lineComment } _, err := w.Write([]byte(line + "\n")) return err } func (te *tomlEncoder) sequenceToInlineArray(seq *CandidateNode) (string, error) { items := make([]string, 0, len(seq.Content)) for _, it := range seq.Content { switch it.Kind { case ScalarNode: items = append(items, te.formatScalar(it)) case SequenceNode: nested, err := te.sequenceToInlineArray(it) if err != nil { return "", err } items = append(items, nested) case MappingNode: inline, err := te.mappingToInlineTable(it) if err != nil { return "", err } items = append(items, inline) default: return "", fmt.Errorf("unsupported array item kind: %v", it.Kind) } } return "[" + strings.Join(items, ", ") + "]", nil } func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) { // key = { a = 1, b = "x" } parts := make([]string, 0, len(m.Content)/2) for i := 0; i < len(m.Content); i += 2 { k := m.Content[i].Value v := m.Content[i+1] switch v.Kind { case ScalarNode: parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v))) case SequenceNode: // inline array in inline table arr, err := te.sequenceToInlineArray(v) if err != nil { return "", err } parts = append(parts, fmt.Sprintf("%s = %s", k, arr)) case MappingNode: // nested inline table inline, err := te.mappingToInlineTable(v) if err != nil { return "", err } parts = append(parts, fmt.Sprintf("%s = %s", k, inline)) default: return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind) } } return "{ " + strings.Join(parts, ", ") + " }", nil } func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *CandidateNode) error { inline, err := te.mappingToInlineTable(m) if err != nil { return err } _, err = w.Write([]byte(key + " = " + inline + "\n")) return err } func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *CandidateNode) error { // Add blank line before table header (or before comment if present) if we wrote root attributes needsBlankLine := te.wroteRootAttr if needsBlankLine { if _, err := w.Write([]byte("\n")); err != nil { return err } te.wroteRootAttr = false // Only add once } // Write head comment before the table header if m.HeadComment != "" { if err := te.writeComment(w, m.HeadComment); err != nil { return err } } // Write table header [a.b.c] header := "[" + strings.Join(path, ".") + "]\n" _, err := w.Write([]byte(header)) return err } // encodeSeparateMapping handles a mapping that should be encoded as table sections. // It emits the table header for this mapping if it has any content, then processes children. func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error { // Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes) hasAttrs := false for i := 0; i < len(m.Content); i += 2 { v := m.Content[i+1] if v.Kind == ScalarNode { hasAttrs = true break } if v.Kind == SequenceNode { // Check if it's NOT an array of tables allMaps := true for _, it := range v.Content { if it.Kind != MappingNode { allMaps = false break } } if !allMaps { hasAttrs = true break } } } // If there are attributes or if the mapping is empty, emit the table header if hasAttrs || len(m.Content) == 0 { if err := te.writeTableHeader(w, path, m); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, path, m); err != nil { return err } return nil } // No attributes, just nested structures - process children for i := 0; i < len(m.Content); i += 2 { k := m.Content[i].Value v := m.Content[i+1] switch v.Kind { case MappingNode: // Emit [path.k] newPath := append(append([]string{}, path...), k) if err := te.writeTableHeader(w, newPath, v); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil { return err } case SequenceNode: // If sequence of maps, emit [[path.k]] per element allMaps := true for _, it := range v.Content { if it.Kind != MappingNode { allMaps = false break } } if allMaps { key := strings.Join(append(append([]string{}, path...), k), ".") for _, it := range v.Content { if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil { return err } } } else { // Regular array attribute under the current table path if err := te.writeArrayAttribute(w, k, v); err != nil { return err } } case ScalarNode: // Attributes directly under the current table path if err := te.writeAttribute(w, k, v); err != nil { return err } } } return nil } func (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool { for i := 0; i < len(m.Content); i += 2 { v := m.Content[i+1] if v.Kind == MappingNode && v.EncodeSeparate { return true } } return false } func (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool { for i := 0; i < len(m.Content); i += 2 { v := m.Content[i+1] // Only consider it structural if mapping has EncodeSeparate or is non-empty if v.Kind == MappingNode && v.EncodeSeparate { return true } if v.Kind == SequenceNode { allMaps := true for _, it := range v.Content { if it.Kind != MappingNode { allMaps = false break } } if allMaps { return true } } } return false } // encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error { // First, attributes (scalars and non-map arrays) for i := 0; i < len(m.Content); i += 2 { k := m.Content[i].Value v := m.Content[i+1] switch v.Kind { case ScalarNode: if err := te.writeAttribute(w, k, v); err != nil { return err } case SequenceNode: allMaps := true for _, it := range v.Content { if it.Kind != MappingNode { allMaps = false break } } if !allMaps { if err := te.writeArrayAttribute(w, k, v); err != nil { return err } } } } // Then, nested arrays of tables with full path for i := 0; i < len(m.Content); i += 2 { k := m.Content[i].Value v := m.Content[i+1] if v.Kind == SequenceNode { allMaps := true for _, it := range v.Content { if it.Kind != MappingNode { allMaps = false break } } if allMaps { dotted := strings.Join(append(append([]string{}, path...), k), ".") for _, it := range v.Content { if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, append(append([]string{}, path...), k), it); err != nil { return err } } } } } // Finally, child mappings that are not marked EncodeSeparate get inlined as attributes for i := 0; i < len(m.Content); i += 2 { k := m.Content[i].Value v := m.Content[i+1] if v.Kind == MappingNode && !v.EncodeSeparate { if err := te.writeInlineTableAttribute(w, k, v); err != nil { return err } } } return nil } // colorizeToml applies syntax highlighting to TOML output using fatih/color func (te *tomlEncoder) colorizeToml(input []byte) []byte { toml := string(input) result := strings.Builder{} // Force color output (don't check for TTY) color.NoColor = false // Create color functions for different token types // Use EnableColor() to ensure colors work even when NO_COLOR env is set commentColorObj := color.New(color.FgHiBlack) commentColorObj.EnableColor() stringColorObj := color.New(color.FgGreen) stringColorObj.EnableColor() numberColorObj := color.New(color.FgHiMagenta) numberColorObj.EnableColor() keyColorObj := color.New(color.FgCyan) keyColorObj.EnableColor() boolColorObj := color.New(color.FgHiMagenta) boolColorObj.EnableColor() sectionColorObj := color.New(color.FgYellow, color.Bold) sectionColorObj.EnableColor() commentColor := commentColorObj.SprintFunc() stringColor := stringColorObj.SprintFunc() numberColor := numberColorObj.SprintFunc() keyColor := keyColorObj.SprintFunc() boolColor := boolColorObj.SprintFunc() sectionColor := sectionColorObj.SprintFunc() // Simple tokenization for TOML colouring i := 0 for i < len(toml) { ch := toml[i] // Comments - from # to end of line if ch == '#' { end := i for end < len(toml) && toml[end] != '\n' { end++ } result.WriteString(commentColor(toml[i:end])) i = end continue } // Table sections - [section] or [[array]] // Only treat '[' as a table section if it appears at the start of the line // (possibly after whitespace). This avoids mis-colouring inline arrays like // "ports = [8000, 8001]" as table sections. if ch == '[' { isSectionHeader := true if i > 0 { isSectionHeader = false j := i - 1 for j >= 0 && toml[j] != '\n' { if toml[j] != ' ' && toml[j] != '\t' && toml[j] != '\r' { // Found a non-whitespace character before this '[' on the same line, // so this is not a table header. break } j-- } if j < 0 || toml[j] == '\n' { // Reached the start of the string or a newline without encountering // any non-whitespace, so '[' is at the logical start of the line. isSectionHeader = true } } if isSectionHeader { end := i + 1 // Check for [[ if end < len(toml) && toml[end] == '[' { end++ } // Find closing ] for end < len(toml) && toml[end] != ']' { end++ } // Include closing ] if end < len(toml) { end++ // Check for ]] if end < len(toml) && toml[end] == ']' { end++ } } result.WriteString(sectionColor(toml[i:end])) i = end continue } } // Strings - quoted text (double or single quotes) if ch == '"' || ch == '\'' { quote := ch end := i + 1 for end < len(toml) { if toml[end] == quote { break } if toml[end] == '\\' && end+1 < len(toml) { // Skip the backslash and the escaped character end += 2 continue } end++ } if end < len(toml) { end++ // include closing quote } result.WriteString(stringColor(toml[i:end])) i = end continue } // Numbers - sequences of digits, possibly with decimal point or minus if (ch >= '0' && ch <= '9') || (ch == '-' && i+1 < len(toml) && toml[i+1] >= '0' && toml[i+1] <= '9') { end := i if ch == '-' { end++ } for end < len(toml) { c := toml[end] if (c >= '0' && c <= '9') || c == '.' || c == 'e' || c == 'E' { end++ } else if (c == '+' || c == '-') && end > 0 && (toml[end-1] == 'e' || toml[end-1] == 'E') { // Only allow + or - immediately after 'e' or 'E' for scientific notation end++ } else { break } } result.WriteString(numberColor(toml[i:end])) i = end continue } // Identifiers/keys - alphanumeric + underscore + dash if (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') || ch == '_' { end := i for end < len(toml) && ((toml[end] >= 'a' && toml[end] <= 'z') || (toml[end] >= 'A' && toml[end] <= 'Z') || (toml[end] >= '0' && toml[end] <= '9') || toml[end] == '_' || toml[end] == '-') { end++ } ident := toml[i:end] // Check if this is a boolean/null keyword switch ident { case "true", "false": result.WriteString(boolColor(ident)) default: // Check if followed by = or whitespace then = (it's a key) j := end for j < len(toml) && (toml[j] == ' ' || toml[j] == '\t') { j++ } if j < len(toml) && toml[j] == '=' { result.WriteString(keyColor(ident)) } else { result.WriteString(ident) // plain text for other identifiers } } i = end continue } // Everything else (whitespace, operators, brackets) - no color result.WriteByte(ch) i++ } return []byte(result.String()) } ================================================ FILE: pkg/yqlib/encoder_uri.go ================================================ //go:build !yq_nouri package yqlib import ( "fmt" "io" "net/url" ) type uriEncoder struct { } func NewUriEncoder() Encoder { return &uriEncoder{} } func (e *uriEncoder) CanHandleAliases() bool { return false } func (e *uriEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error { return nil } func (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error { if node.guessTagFromCustomType() != "!!str" { return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag) } _, err := writer.Write([]byte(url.QueryEscape(node.Value))) return err } ================================================ FILE: pkg/yqlib/encoder_xml.go ================================================ //go:build !yq_noxml package yqlib import ( "encoding/xml" "fmt" "io" "regexp" "strings" ) type xmlEncoder struct { indentString string writer io.Writer prefs XmlPreferences leadingContent string } func NewXMLEncoder(prefs XmlPreferences) Encoder { var indentString = "" for index := 0; index < prefs.Indent; index++ { indentString = indentString + " " } return &xmlEncoder{indentString, nil, prefs, ""} } func (e *xmlEncoder) CanHandleAliases() bool { return false } func (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error { return nil } func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error { e.leadingContent = content return nil } func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error { encoder := xml.NewEncoder(writer) // hack so we can manually add newlines to procInst and directives e.writer = writer encoder.Indent("", e.indentString) var newLine xml.CharData = []byte("\n") if node.Tag == "!!map" { // make sure processing instructions are encoded first for i := 0; i < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] if key.Value == (e.prefs.ProcInstPrefix + "xml") { name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} if err := encoder.EncodeToken(procInst); err != nil { return err } if _, err := e.writer.Write([]byte("\n")); err != nil { log.Warning("Unable to write newline, skipping: %w", err) } } } } if e.leadingContent != "" { // remove first and last newlines if present err := e.encodeComment(encoder, e.leadingContent) if err != nil { return err } err = encoder.EncodeToken(newLine) if err != nil { return err } } switch node.Kind { case MappingNode: err := e.encodeTopLevelMap(encoder, node) if err != nil { return err } case ScalarNode: var charData xml.CharData = []byte(node.Value) err := encoder.EncodeToken(charData) if err != nil { return err } return encoder.Flush() default: return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", node.Tag) } return encoder.EncodeToken(newLine) } func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode) error { err := e.encodeComment(encoder, headAndLineComment(node)) if err != nil { return err } for i := 0; i < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] start := xml.StartElement{Name: xml.Name{Local: key.Value}} log.Debugf("comments of key %v", key.Value) err := e.encodeComment(encoder, headAndLineComment(key)) if err != nil { return err } if headAndLineComment(key) != "" { var newLine xml.CharData = []byte("\n") err = encoder.EncodeToken(newLine) if err != nil { return err } } if key.Value == (e.prefs.ProcInstPrefix + "xml") { //nolint // dont double process these. } else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) { name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} if err := encoder.EncodeToken(procInst); err != nil { return err } if _, err := e.writer.Write([]byte("\n")); err != nil { log.Warning("Unable to write newline, skipping: %w", err) } } else if key.Value == e.prefs.DirectiveName { var directive xml.Directive = []byte(value.Value) if err := encoder.EncodeToken(directive); err != nil { return err } if _, err := e.writer.Write([]byte("\n")); err != nil { log.Warning("Unable to write newline, skipping: %w", err) } } else { log.Debugf("recursing") err = e.doEncode(encoder, value, start) if err != nil { return err } } err = e.encodeComment(encoder, footComment(key)) if err != nil { return err } } return e.encodeComment(encoder, footComment(node)) } func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error { err := encoder.EncodeToken(start) if err != nil { return err } return e.encodeComment(encoder, headComment(node)) } func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error { err := encoder.EncodeToken(start.End()) if err != nil { return err } return e.encodeComment(encoder, footComment(node)) } func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error { switch node.Kind { case MappingNode: return e.encodeMap(encoder, node, start) case SequenceNode: return e.encodeArray(encoder, node, start) case ScalarNode: err := e.encodeStart(encoder, node, start) if err != nil { return err } var charData xml.CharData = []byte(node.Value) err = encoder.EncodeToken(charData) if err != nil { return err } if err = e.encodeComment(encoder, lineComment(node)); err != nil { return err } return e.encodeEnd(encoder, node, start) } return fmt.Errorf("unsupported type %v", node.Tag) } var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`) var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`) var chompRegexp = regexp.MustCompile(`\n$`) func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error { if commentStr != "" { log.Debugf("got comment [%v]", commentStr) // multi line string if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") { commentStr = chompRegexp.ReplaceAllString(commentStr, "") log.Debugf("chompRegexp [%v]", commentStr) commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2") log.Debugf("processed multiline [%v]", commentStr) // if the first line is non blank, add a space if commentStr[0] != '\n' && commentStr[0] != ' ' { commentStr = " " + commentStr } } else { commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1") } if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") { commentStr = commentStr + " " log.Debugf("added suffix [%v]", commentStr) } log.Debugf("encoding comment [%v]", commentStr) var comment xml.Comment = []byte(commentStr) err := encoder.EncodeToken(comment) if err != nil { return err } } return nil } func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error { if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil { return err } for i := 0; i < len(node.Content); i++ { value := node.Content[i] if err := e.doEncode(encoder, value, start.Copy()); err != nil { return err } } return e.encodeComment(encoder, footComment(node)) } func (e *xmlEncoder) isAttribute(name string) bool { return strings.HasPrefix(name, e.prefs.AttributePrefix) && name != e.prefs.ContentName && name != e.prefs.DirectiveName && !strings.HasPrefix(name, e.prefs.ProcInstPrefix) } func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error { log.Debug("its a map") //first find all the attributes and put them on the start token for i := 0; i < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] if e.isAttribute(key.Value) { if value.Kind == ScalarNode { attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1) start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value}) } else { return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag) } } } err := e.encodeStart(encoder, node, start) if err != nil { return err } //now we encode non attribute tokens for i := 0; i < len(node.Content); i += 2 { key := node.Content[i] value := node.Content[i+1] err := e.encodeComment(encoder, headAndLineComment(key)) if err != nil { return err } if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) { name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1) procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)} if err := encoder.EncodeToken(procInst); err != nil { return err } } else if key.Value == e.prefs.DirectiveName { var directive xml.Directive = []byte(value.Value) if err := encoder.EncodeToken(directive); err != nil { return err } } else if key.Value == e.prefs.ContentName { // directly encode the contents err = e.encodeComment(encoder, headAndLineComment(value)) if err != nil { return err } var charData xml.CharData = []byte(value.Value) err = encoder.EncodeToken(charData) if err != nil { return err } err = e.encodeComment(encoder, footComment(value)) if err != nil { return err } } else if !e.isAttribute(key.Value) { start := xml.StartElement{Name: xml.Name{Local: key.Value}} err := e.doEncode(encoder, value, start) if err != nil { return err } } err = e.encodeComment(encoder, footComment(key)) if err != nil { return err } } return e.encodeEnd(encoder, node, start) } ================================================ FILE: pkg/yqlib/encoder_yaml.go ================================================ package yqlib import ( "bytes" "io" "strings" "go.yaml.in/yaml/v4" ) type yamlEncoder struct { prefs YamlPreferences } func NewYamlEncoder(prefs YamlPreferences) Encoder { return &yamlEncoder{prefs} } func (ye *yamlEncoder) CanHandleAliases() bool { return true } func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { return PrintYAMLDocumentSeparator(writer, ye.prefs.PrintDocSeparators) } func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { return PrintYAMLLeadingContent(writer, content, ye.prefs.PrintDocSeparators, ye.prefs.ColorsEnabled) } func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { log.Debug("encoderYaml - going to print %v", NodeToString(node)) // Detect line ending style from LeadingContent lineEnding := "\n" if strings.Contains(node.LeadingContent, "\r\n") { lineEnding = "\r\n" } if node.Kind == ScalarNode && ye.prefs.UnwrapScalar { valueToPrint := node.Value if node.LeadingContent == "" || valueToPrint != "" { valueToPrint = valueToPrint + lineEnding } return writeString(writer, valueToPrint) } destination := writer tempBuffer := bytes.NewBuffer(nil) if ye.prefs.ColorsEnabled { destination = tempBuffer } var encoder = yaml.NewEncoder(destination) encoder.SetIndent(ye.prefs.Indent) if ye.prefs.CompactSequenceIndent { encoder.CompactSeqIndent() } target, err := node.MarshalYAML() if err != nil { return err } trailingContent := target.FootComment target.FootComment = "" if err := encoder.Encode(target); err != nil { return err } if err := ye.PrintLeadingContent(destination, trailingContent); err != nil { return err } if ye.prefs.ColorsEnabled { return colorizeAndPrint(tempBuffer.Bytes(), writer) } return nil } ================================================ FILE: pkg/yqlib/expression_parser.go ================================================ package yqlib import ( "fmt" "strings" ) type ExpressionNode struct { Operation *Operation LHS *ExpressionNode RHS *ExpressionNode Parent *ExpressionNode } type ExpressionParserInterface interface { ParseExpression(expression string) (*ExpressionNode, error) } type expressionParserImpl struct { pathTokeniser expressionTokeniser pathPostFixer expressionPostFixer } func newExpressionParser() ExpressionParserInterface { return &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()} } func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) { log.Debug("Parsing expression: [%v]", expression) tokens, err := p.pathTokeniser.Tokenise(expression) if err != nil { return nil, err } var Operations []*Operation Operations, err = p.pathPostFixer.ConvertToPostfix(tokens) if err != nil { return nil, err } return p.createExpressionTree(Operations) } func (p *expressionParserImpl) createExpressionTree(postFixPath []*Operation) (*ExpressionNode, error) { var stack = make([]*ExpressionNode, 0) if len(postFixPath) == 0 { return nil, nil } for _, Operation := range postFixPath { var newNode = ExpressionNode{Operation: Operation} log.Debugf("pathTree %v ", Operation.toString()) if Operation.OperationType.NumArgs > 0 { numArgs := Operation.OperationType.NumArgs switch numArgs { case 1: if len(stack) < 1 { // Allow certain unary ops to accept zero args by interpreting missing RHS as nil // TODO - make this more general on OperationType if Operation.OperationType == firstOpType { // no RHS provided; proceed without popping break } return nil, fmt.Errorf("'%v' expects 1 arg but received none", strings.TrimSpace(Operation.StringValue)) } remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] newNode.RHS = rhs rhs.Parent = &newNode stack = remaining case 2: if len(stack) < 2 { return nil, fmt.Errorf("'%v' expects 2 args but there is %v", strings.TrimSpace(Operation.StringValue), len(stack)) } remaining, lhs, rhs := stack[:len(stack)-2], stack[len(stack)-2], stack[len(stack)-1] newNode.LHS = lhs lhs.Parent = &newNode newNode.RHS = rhs rhs.Parent = &newNode stack = remaining } } stack = append(stack, &newNode) } if len(stack) != 1 { return nil, fmt.Errorf("bad expression, please check expression syntax") } return stack[0], nil } ================================================ FILE: pkg/yqlib/expression_parser_test.go ================================================ package yqlib import ( "testing" "github.com/mikefarah/yq/v4/test" ) func getExpressionParser() ExpressionParserInterface { InitExpressionParser() return ExpressionParser } func TestParserCreateMapColonOnItsOwn(t *testing.T) { _, err := getExpressionParser().ParseExpression(":") test.AssertResultComplex(t, "':' expects 2 args but there is 0", err.Error()) } func TestParserNoMatchingCloseBracket(t *testing.T) { _, err := getExpressionParser().ParseExpression(".cat | with(.;.bob") test.AssertResultComplex(t, "bad expression - probably missing close bracket on WITH", err.Error()) } func TestParserNoMatchingCloseCollect(t *testing.T) { _, err := getExpressionParser().ParseExpression("[1,2") test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error()) } func TestParserNoMatchingCloseObjectInCollect(t *testing.T) { _, err := getExpressionParser().ParseExpression(`[{"b": "c"]`) test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error()) } func TestParserNoMatchingCloseInCollect(t *testing.T) { _, err := getExpressionParser().ParseExpression(`[(.a]`) test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error()) } func TestParserNoMatchingCloseCollectObject(t *testing.T) { _, err := getExpressionParser().ParseExpression(`{"a": "b"`) test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error()) } func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) { _, err := getExpressionParser().ParseExpression(`{"b": [1}`) test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error()) } func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) { _, err := getExpressionParser().ParseExpression(`{"b": (1}`) test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error()) } func TestParserNoArgsForTwoArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression("=") test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error()) } func TestParserOneLhsArgsForTwoArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression(".a =") test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) } func TestParserOneRhsArgsForTwoArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression("= .a") test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) } func TestParserTwoArgsForTwoArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression(".a = .b") test.AssertResultComplex(t, nil, err) } func TestParserNoArgsForOneArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression("explode") test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error()) } func TestParserOneArgForOneArgOp(t *testing.T) { _, err := getExpressionParser().ParseExpression("explode(.)") test.AssertResultComplex(t, nil, err) } func TestParserExtraArgs(t *testing.T) { _, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)") test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error()) } func TestParserEmptyExpression(t *testing.T) { _, err := getExpressionParser().ParseExpression("") test.AssertResultComplex(t, nil, err) } func TestParserSingleOperation(t *testing.T) { result, err := getExpressionParser().ParseExpression(".") test.AssertResultComplex(t, nil, err) if result == nil { t.Fatal("Expected non-nil result for single operation") } if result.Operation == nil { t.Fatal("Expected operation to be set") } } func TestParserFirstOpWithZeroArgs(t *testing.T) { // Test the special case where firstOpType can accept zero args result, err := getExpressionParser().ParseExpression("first") test.AssertResultComplex(t, nil, err) if result == nil { t.Fatal("Expected non-nil result for first operation with zero args") } } func TestParserInvalidExpressionTree(t *testing.T) { // This tests the createExpressionTree function with malformed postfix parser := getExpressionParser().(*expressionParserImpl) // Create invalid postfix operations that would leave more than one item on stack invalidOps := []*Operation{ {OperationType: &operationType{NumArgs: 0}}, {OperationType: &operationType{NumArgs: 0}}, } _, err := parser.createExpressionTree(invalidOps) test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error()) } ================================================ FILE: pkg/yqlib/expression_postfix.go ================================================ package yqlib import ( "errors" "fmt" logging "gopkg.in/op/go-logging.v1" ) type expressionPostFixer interface { ConvertToPostfix([]*token) ([]*Operation, error) } type expressionPostFixerImpl struct { } func newExpressionPostFixer() expressionPostFixer { return &expressionPostFixerImpl{} } func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operation) { var newOp *token opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1] log.Debugf("popped %v from opstack to results", newOp.toString(true)) return opStack, append(result, newOp.Operation) } func validateNoOpenTokens(token *token) error { switch token.TokenType { case openCollect: return fmt.Errorf(("bad expression, could not find matching `]`")) case openCollectObject: return fmt.Errorf(("bad expression, could not find matching `}`")) case openBracket: return fmt.Errorf(("bad expression, could not find matching `)`")) } return nil } func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) { var result []*Operation // surround the whole thing with brackets var opStack = []*token{{TokenType: openBracket}} var tokens = append(infixTokens, &token{TokenType: closeBracket}) for _, currentToken := range tokens { log.Debugf("postfix processing currentToken %v", currentToken.toString(true)) switch currentToken.TokenType { case openBracket, openCollect, openCollectObject: opStack = append(opStack, currentToken) log.Debugf("put %v onto the opstack", currentToken.toString(true)) case closeCollect, closeCollectObject: var opener tokenType = openCollect var collectOperator = collectOpType if currentToken.TokenType == closeCollectObject { opener = openCollectObject collectOperator = collectObjectOpType } for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener { missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1]) if missingClosingTokenErr != nil { return nil, missingClosingTokenErr } opStack, result = popOpToResult(opStack, result) } if len(opStack) == 0 { return nil, errors.New("bad path expression, got close collect brackets without matching opening bracket") } // now we should have [ as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] log.Debugf("deleting open bracket from opstack") //and append a collect to the result // hack - see if there's the optional traverse flag // on the close op - move it to the traverse array op // allows for .["cat"]? prefs := traversePreferences{} closeTokenMatch := currentToken.Match if closeTokenMatch[len(closeTokenMatch)-1:] == "?" { prefs.OptionalTraverse = true } result = append(result, &Operation{OperationType: collectOperator}) log.Debugf("put collect onto the result") if opener != openCollect { result = append(result, &Operation{OperationType: shortPipeOpType}) log.Debugf("put shortpipe onto the result") } //traverseArrayCollect is a sneaky op that needs to be included too //when closing a ] if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType { opStack[len(opStack)-1].Operation.Preferences = prefs opStack, result = popOpToResult(opStack, result) } case closeBracket: for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket { missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1]) if missingClosingTokenErr != nil { return nil, missingClosingTokenErr } opStack, result = popOpToResult(opStack, result) } if len(opStack) == 0 { return nil, errors.New("bad expression, got close brackets without matching opening bracket") } // now we should have ( as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] default: var currentPrecedence = currentToken.Operation.OperationType.Precedence // pop off higher precedent operators onto the result for len(opStack) > 0 && opStack[len(opStack)-1].TokenType == operationToken && opStack[len(opStack)-1].Operation.OperationType.Precedence > currentPrecedence { opStack, result = popOpToResult(opStack, result) } // add this operator to the opStack opStack = append(opStack, currentToken) log.Debugf("put %v onto the opstack", currentToken.toString(true)) } } log.Debugf("opstackLen: %v", len(opStack)) if len(opStack) > 0 { log.Debugf("opstack:") for _, token := range opStack { log.Debugf("- %v", token.toString(true)) } return nil, fmt.Errorf("bad expression - probably missing close bracket on %v", opStack[len(opStack)-1].toString(false)) } if log.IsEnabledFor(logging.DEBUG) { log.Debugf("PostFix Result:") for _, currentToken := range result { log.Debugf("> %v", currentToken.toString()) } } return result, nil } ================================================ FILE: pkg/yqlib/expression_processing_test.go ================================================ package yqlib import ( "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) var variableWithNewLine = `"cat "` var pathTests = []struct { path string expectedTokens []interface{} expectedPostFix []interface{} }{ { `envsubst(ne)`, append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), }, { `envsubst(nu)`, append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), }, { `envsubst(nu, ne)`, append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), }, { `envsubst(ne, nu)`, append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), }, { `[.a, .b]`, append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"), append(make([]interface{}, 0), "a", "b", "UNION", "COLLECT"), }, { `.[env(myenv)]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "ENV", "]"), append(make([]interface{}, 0), "SELF", "ENV", "COLLECT", "TRAVERSE_ARRAY"), }, { `.["cat"].["dog"]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"), append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), }, { `.["cat"]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"), append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY"), }, { "with(.a;.=3)", append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"), append(make([]interface{}, 0), "a", "SELF", "3 (int64)", "ASSIGN", "BLOCK", "WITH"), }, { "0x12", append(make([]interface{}, 0), "18 (int64)"), append(make([]interface{}, 0), "18 (int64)"), }, { "0X12", append(make([]interface{}, 0), "18 (int64)"), append(make([]interface{}, 0), "18 (int64)"), }, { ".a\n", append(make([]interface{}, 0), "a"), append(make([]interface{}, 0), "a"), }, { variableWithNewLine, append(make([]interface{}, 0), "cat\n (string)"), append(make([]interface{}, 0), "cat\n (string)"), }, { `.[0]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), }, { `.[0][1]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"), append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "TRAVERSE_ARRAY"), }, { `"\""`, append(make([]interface{}, 0), "\" (string)"), append(make([]interface{}, 0), "\" (string)"), }, { `[]|join(".")`, append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"), append(make([]interface{}, 0), "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE"), }, { `{"cool": .b or .c}`, append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"), append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `{"cool": []|join(".")}`, append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"), append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"), append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"), }, { `.a | .b | .c`, append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"), append(make([]interface{}, 0), "a", "b", "c", "PIPE", "PIPE"), }, { `[]`, append(make([]interface{}, 0), "[", "EMPTY", "]"), append(make([]interface{}, 0), "EMPTY", "COLLECT"), }, { `{}`, append(make([]interface{}, 0), "{", "EMPTY", "}"), append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `[{}]`, append(make([]interface{}, 0), "[", "{", "EMPTY", "}", "]"), append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT"), }, { `.realnames as $names | $names["anon"]`, append(make([]interface{}, 0), "realnames", "ASSIGN_VARIABLE", "GET_VARIABLE", "PIPE", "GET_VARIABLE", "TRAVERSE_ARRAY", "[", "anon (string)", "]"), append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "TRAVERSE_ARRAY", "PIPE"), }, { `.b[.a]`, append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"), append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"), }, { `.b[.a]?`, append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"), append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"), }, { `.[]`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), }, { `.a[]`, append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), }, { `.a[]?`, append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"), }, { `.a.[]`, append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"), append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), }, { `.a[0]`, append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), }, { `.a[0]?`, append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"), }, { `.a.[0]`, append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"), append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"), }, { `.a[].c`, append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "c"), append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"), }, { `[3]`, append(make([]interface{}, 0), "[", "3 (int64)", "]"), append(make([]interface{}, 0), "3 (int64)", "COLLECT"), }, { `.key.array + .key.array2`, append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ADD", "key", "SHORT_PIPE", "array2"), append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ADD"), }, { `.key.array * .key.array2`, append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "MULTIPLY", "key", "SHORT_PIPE", "array2"), append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "MULTIPLY"), }, { `.key.array // .key.array2`, append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ALTERNATIVE", "key", "SHORT_PIPE", "array2"), append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ALTERNATIVE"), }, { `.a | .[].b == "apple"`, append(make([]interface{}, 0), "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"), append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"), }, { `(.a | .[].b) == "apple"`, append(make([]interface{}, 0), "(", "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"), append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"), }, { `.[] | select(. == "*at")`, append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"), append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"), }, { `[true]`, append(make([]interface{}, 0), "[", "true (bool)", "]"), append(make([]interface{}, 0), "true (bool)", "COLLECT"), }, { `[true, false]`, append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"), append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT"), }, { `"mike": .a`, append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"), append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP"), }, { `.a: "mike"`, append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"), append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"), }, { `{"mike": .a}`, append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"), append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `{.a: "mike"}`, append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"), append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `{.a: .c, .b.[]: .f.g[]}`, append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"), append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"), }, { `explode(.a.b)`, append(make([]interface{}, 0), "EXPLODE", "(", "a", "SHORT_PIPE", "b", ")"), append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "EXPLODE"), }, { `.a.b style="folded"`, append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_STYLE", "folded (string)"), append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "folded (string)", "ASSIGN_STYLE"), }, { `tag == "str"`, append(make([]interface{}, 0), "GET_TAG", "EQUALS", "str (string)"), append(make([]interface{}, 0), "GET_TAG", "str (string)", "EQUALS"), }, { `. tag= "str"`, append(make([]interface{}, 0), "SELF", "ASSIGN_TAG", "str (string)"), append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_TAG"), }, { `lineComment == "str"`, append(make([]interface{}, 0), "GET_COMMENT", "EQUALS", "str (string)"), append(make([]interface{}, 0), "GET_COMMENT", "str (string)", "EQUALS"), }, { `. lineComment= "str"`, append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"), append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"), }, { `. lineComment |= "str"`, append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"), append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"), }, { `.a.b tag="!!str"`, append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"), append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "!!str (string)", "ASSIGN_TAG"), }, { `""`, append(make([]interface{}, 0), " (string)"), append(make([]interface{}, 0), " (string)"), }, { `.foo* | (. style="flow")`, append(make([]interface{}, 0), "foo*", "PIPE", "(", "SELF", "ASSIGN_STYLE", "flow (string)", ")"), append(make([]interface{}, 0), "foo*", "SELF", "flow (string)", "ASSIGN_STYLE", "PIPE"), }, } var tokeniser = newParticipleLexer() var postFixer = newExpressionPostFixer() func TestPathParsing(t *testing.T) { for _, tt := range pathTests { tokens, err := tokeniser.Tokenise(tt.path) if err != nil { t.Error(tt.path, err) } var tokenValues []interface{} for _, token := range tokens { tokenValues = append(tokenValues, token.toString(false)) } test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path)) results, errorP := postFixer.ConvertToPostfix(tokens) var readableResults []interface{} for _, token := range results { readableResults = append(readableResults, token.toString()) } if errorP != nil { t.Error(tt.path, err) } test.AssertResultComplexWithContext(t, tt.expectedPostFix, readableResults, fmt.Sprintf("postfix: %v", tt.path)) } } ================================================ FILE: pkg/yqlib/file_utils.go ================================================ package yqlib import ( "fmt" "io" "os" ) func tryRenameFile(from string, to string) error { if info, err := os.Lstat(to); err == nil && info.Mode()&os.ModeSymlink != 0 { log.Debug("Target file is symlink, skipping rename and attempting to copy contents") if copyError := copyFileContents(from, to); copyError != nil { return fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError) } tryRemoveTempFile(from) return nil } else if renameError := os.Rename(from, to); renameError != nil { log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) log.Debug(renameError.Error()) log.Debug("going to try copying instead") // can't do this rename when running in docker to a file targeted in a mounted volume, // so gracefully degrade to copying the entire contents. if copyError := copyFileContents(from, to); copyError != nil { return fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError) } tryRemoveTempFile(from) } return nil } func tryRemoveTempFile(filename string) { log.Debug("Removing temp file: %v", filename) removeErr := os.Remove(filename) if removeErr != nil { log.Errorf("Failed to remove temp file: %v", filename) } } // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang func copyFileContents(src, dst string) (err error) { // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside thar directory. in, err := os.Open(src) // #nosec if err != nil { return err } defer safelyCloseFile(in) out, err := os.Create(dst) // #nosec if err != nil { return err } defer safelyCloseFile(out) if _, err = io.Copy(out, in); err != nil { return err } return out.Sync() } func SafelyCloseReader(reader io.Reader) { switch reader := reader.(type) { case *os.File: safelyCloseFile(reader) } } func safelyCloseFile(file *os.File) { err := file.Close() if err != nil { log.Error("Error closing file!") log.Error(err.Error()) } } func createTempFile() (*os.File, error) { _, err := os.Stat(os.TempDir()) if os.IsNotExist(err) { err = os.Mkdir(os.TempDir(), 0700) if err != nil { return nil, err } } else if err != nil { return nil, err } file, err := os.CreateTemp("", "temp") if err != nil { return nil, err } return file, err } ================================================ FILE: pkg/yqlib/format.go ================================================ package yqlib import ( "fmt" "path/filepath" "slices" "strings" ) type EncoderFactoryFunction func() Encoder type DecoderFactoryFunction func() Decoder type Format struct { FormalName string Names []string EncoderFactory EncoderFactoryFunction DecoderFactory DecoderFactoryFunction } var YamlFormat = &Format{"yaml", []string{"y", "yml"}, func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) }, func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) }, } var KYamlFormat = &Format{"kyaml", []string{"ky"}, func() Encoder { return NewKYamlEncoder(ConfiguredKYamlPreferences) }, // KYaml is stricter YAML func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) }, } var JSONFormat = &Format{"json", []string{"j"}, func() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) }, func() Decoder { return NewJSONDecoder() }, } var PropertiesFormat = &Format{"props", []string{"p", "properties"}, func() Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) }, func() Decoder { return NewPropertiesDecoder() }, } var CSVFormat = &Format{"csv", []string{"c"}, func() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) }, func() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) }, } var TSVFormat = &Format{"tsv", []string{"t"}, func() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) }, func() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) }, } var XMLFormat = &Format{"xml", []string{"x"}, func() Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) }, func() Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) }, } var Base64Format = &Format{"base64", []string{}, func() Encoder { return NewBase64Encoder() }, func() Decoder { return NewBase64Decoder() }, } var UriFormat = &Format{"uri", []string{}, func() Encoder { return NewUriEncoder() }, func() Decoder { return NewUriDecoder() }, } var ShFormat = &Format{"", nil, func() Encoder { return NewShEncoder() }, nil, } var TomlFormat = &Format{"toml", []string{}, func() Encoder { return NewTomlEncoderWithPrefs(ConfiguredTomlPreferences) }, func() Decoder { return NewTomlDecoder() }, } var HclFormat = &Format{"hcl", []string{"h", "tf"}, func() Encoder { return NewHclEncoder(ConfiguredHclPreferences) }, func() Decoder { return NewHclDecoder() }, } var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, func() Encoder { return NewShellVariablesEncoder() }, nil, } var LuaFormat = &Format{"lua", []string{"l"}, func() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) }, func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) }, } var INIFormat = &Format{"ini", []string{"i"}, func() Encoder { return NewINIEncoder() }, func() Decoder { return NewINIDecoder() }, } var Formats = []*Format{ YamlFormat, KYamlFormat, JSONFormat, PropertiesFormat, CSVFormat, TSVFormat, XMLFormat, Base64Format, UriFormat, ShFormat, TomlFormat, HclFormat, ShellVariablesFormat, LuaFormat, INIFormat, } func (f *Format) MatchesName(name string) bool { if f.FormalName == name { return true } return slices.Contains(f.Names, name) } func (f *Format) GetConfiguredEncoder() Encoder { return f.EncoderFactory() } func FormatStringFromFilename(filename string) string { if filename != "" { GetLogger().Debugf("checking filename '%s' for auto format detection", filename) ext := filepath.Ext(filename) if len(ext) >= 2 && ext[0] == '.' { format := strings.ToLower(ext[1:]) GetLogger().Debugf("detected format '%s'", format) return format } } GetLogger().Debugf("using default inputFormat 'yaml'") return "yaml" } func FormatFromString(format string) (*Format, error) { if format != "" { for _, printerFormat := range Formats { if printerFormat.MatchesName(format) { return printerFormat, nil } } } return nil, fmt.Errorf("unknown format '%v' please use [%v]", format, GetAvailableOutputFormatString()) } func GetAvailableOutputFormats() []*Format { var formats = []*Format{} for _, printerFormat := range Formats { if printerFormat.EncoderFactory != nil { formats = append(formats, printerFormat) } } return formats } func GetAvailableOutputFormatString() string { var formats = []string{} for _, printerFormat := range GetAvailableOutputFormats() { if printerFormat.FormalName != "" { formats = append(formats, printerFormat.FormalName) } if len(printerFormat.Names) >= 1 { formats = append(formats, printerFormat.Names[0]) } } return strings.Join(formats, "|") } func GetAvailableInputFormats() []*Format { var formats = []*Format{} for _, printerFormat := range Formats { if printerFormat.DecoderFactory != nil { formats = append(formats, printerFormat) } } return formats } func GetAvailableInputFormatString() string { var formats = []string{} for _, printerFormat := range GetAvailableInputFormats() { if printerFormat.FormalName != "" { formats = append(formats, printerFormat.FormalName) } if len(printerFormat.Names) >= 1 { formats = append(formats, printerFormat.Names[0]) } } return strings.Join(formats, "|") } ================================================ FILE: pkg/yqlib/format_test.go ================================================ package yqlib import ( "fmt" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) type formatStringScenario struct { description string input string expectedFormat *Format expectedError string } var formatStringScenarios = []formatStringScenario{ { description: "yaml", input: "yaml", expectedFormat: YamlFormat, }, { description: "Unknown format type", input: "doc", expectedError: "unknown format 'doc' please use", }, { description: "blank should error", input: "", expectedError: "unknown format '' please use", }, } func TestFormatFromString(t *testing.T) { for _, tt := range formatStringScenarios { actualFormat, actualError := FormatFromString(tt.input) if tt.expectedError != "" { if actualError == nil { t.Errorf("Expected [%v] error but found none", tt.expectedError) } else { test.AssertResultWithContext(t, true, strings.Contains(actualError.Error(), tt.expectedError), fmt.Sprintf("Expected [%v] to contain [%v]", actualError.Error(), tt.expectedError), ) } } else { test.AssertResult(t, tt.expectedFormat, actualFormat) } } } func TestFormatStringFromFilename(t *testing.T) { test.AssertResult(t, "yaml", FormatStringFromFilename("test.Yaml")) test.AssertResult(t, "yaml", FormatStringFromFilename("test.index.Yaml")) test.AssertResult(t, "yaml", FormatStringFromFilename("test")) test.AssertResult(t, "json", FormatStringFromFilename("test.json")) test.AssertResult(t, "json", FormatStringFromFilename("TEST.JSON")) test.AssertResult(t, "yaml", FormatStringFromFilename("test.json/foo")) test.AssertResult(t, "yaml", FormatStringFromFilename("")) } ================================================ FILE: pkg/yqlib/formatting_expressions_test.go ================================================ package yqlib import ( "bufio" "fmt" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) var formattingExpressionScenarios = []formatScenario{ { description: "Using expression files and comments", skipDoc: true, input: "a:\n b: old", expression: "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n", expected: "a:\n b: new\n c: frog\n", }, { description: "Using expression files and comments", subdescription: "Note that you can execute the file directly - but make sure you make the expression file executable.", input: "a:\n b: old", expression: "#! yq\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n", expected: "a:\n b: new\n c: frog\n", scenarioType: "shebang", }, { description: "Flags in expression files", subdescription: "You can specify flags on the shebang line, this only works when executing the file directly.", input: "a:\n b: old", expression: "#! yq -oj\n\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n| .a.c = \"frog\"\n\n# Now good things will happen.\n", expected: "a:\n b: new\n c: frog\n", scenarioType: "shebang-json", }, { description: "Commenting out yq expressions", subdescription: "Note that `c` is no longer set to 'frog'. In this example we're calling yq directly and passing the expression file into `--from-file`, this is no different from executing the expression file directly.", input: "a:\n b: old", expression: "#! yq\n# This is a yq expression that updates the map\n# for several great reasons outlined here.\n\n.a.b = \"new\" # line comment here\n# | .a.c = \"frog\"\n\n# Now good things will happen.\n", expected: "a:\n b: new\n", }, } func documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yaml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "And an 'update.yq' expression file of:\n") writeOrPanic(w, fmt.Sprintf("```bash\n%v```\n", s.expression)) writeOrPanic(w, "then\n") if strings.HasPrefix(s.scenarioType, "shebang") { writeOrPanic(w, "```bash\n./update.yq sample.yaml\n```\n") } else { writeOrPanic(w, "```bash\nyq --from-file update.yq sample.yml\n```\n") } writeOrPanic(w, "will output\n") encoder := NewYamlEncoder(ConfiguredYamlPreferences) if s.scenarioType == "shebang-json" { encoder = NewJSONEncoder(ConfiguredJSONPreferences) } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), encoder))) } func TestExpressionCommentScenarios(t *testing.T) { for _, tt := range formattingExpressionScenarios { test.AssertResultComplexWithContext(t, tt.expected, mustProcessFormatScenario(tt, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), tt.description) } genericScenarios := make([]interface{}, len(formattingExpressionScenarios)) for i, s := range formattingExpressionScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "formatting-expressions", genericScenarios, documentExpressionScenario) } ================================================ FILE: pkg/yqlib/front_matter.go ================================================ package yqlib import ( "bufio" "errors" "io" "os" ) type frontMatterHandler interface { Split() error GetYamlFrontMatterFilename() string GetContentReader() io.Reader CleanUp() } type frontMatterHandlerImpl struct { originalFilename string yamlFrontMatterFilename string contentReader io.Reader } func NewFrontMatterHandler(originalFilename string) frontMatterHandler { return &frontMatterHandlerImpl{originalFilename, "", nil} } func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string { return f.yamlFrontMatterFilename } func (f *frontMatterHandlerImpl) GetContentReader() io.Reader { return f.contentReader } func (f *frontMatterHandlerImpl) CleanUp() { tryRemoveTempFile(f.yamlFrontMatterFilename) } // Splits the given file by yaml front matter // yaml content will be saved to first temporary file // remaining content will be saved to second temporary file func (f *frontMatterHandlerImpl) Split() error { var reader *bufio.Reader var err error if f.originalFilename == "-" { reader = bufio.NewReader(os.Stdin) } else { file, err := os.Open(f.originalFilename) // #nosec if err != nil { return err } reader = bufio.NewReader(file) } f.contentReader = reader yamlTempFile, err := createTempFile() if err != nil { return err } f.yamlFrontMatterFilename = yamlTempFile.Name() log.Debug("yamlTempFile: %v", yamlTempFile.Name()) lineCount := 0 for { peekBytes, err := reader.Peek(3) if errors.Is(err, io.EOF) { // we've finished reading the yaml content..I guess break } else if err != nil { return err } if lineCount > 0 && string(peekBytes) == "---" { // we've finished reading the yaml content.. break } line, errReading := reader.ReadString('\n') lineCount = lineCount + 1 if errReading != nil && !errors.Is(errReading, io.EOF) { return errReading } _, errWriting := yamlTempFile.WriteString(line) if errWriting != nil { return errWriting } } safelyCloseFile(yamlTempFile) return nil } ================================================ FILE: pkg/yqlib/front_matter_test.go ================================================ package yqlib import ( "io" "os" "testing" "github.com/mikefarah/yq/v4/test" ) func createTestFile(content string) string { tempFile, err := createTempFile() if err != nil { panic(err) } _, err = tempFile.Write([]byte(content)) if err != nil { panic(err) } safelyCloseFile(tempFile) return tempFile.Name() } func readFile(filename string) string { bytes, err := os.ReadFile(filename) if err != nil { panic(err) } return string(bytes) } func TestFrontMatterSplitWithLeadingSep(t *testing.T) { file := createTestFile(`--- a: apple b: banana --- not a yaml: doc `) expectedYamlFm := `--- a: apple b: banana ` expectedContent := `--- not a yaml: doc ` fmHandler := NewFrontMatterHandler(file) err := fmHandler.Split() if err != nil { panic(err) } yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) test.AssertResult(t, expectedYamlFm, yamlFm) contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) if err != nil { panic(err) } test.AssertResult(t, expectedContent, string(contentBytes)) tryRemoveTempFile(file) fmHandler.CleanUp() } func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) { file := createTestFile(`a: apple b: banana --- not a yaml: doc `) expectedYamlFm := `a: apple b: banana ` expectedContent := `--- not a yaml: doc ` fmHandler := NewFrontMatterHandler(file) err := fmHandler.Split() if err != nil { panic(err) } yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) test.AssertResult(t, expectedYamlFm, yamlFm) contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) if err != nil { panic(err) } test.AssertResult(t, expectedContent, string(contentBytes)) tryRemoveTempFile(file) fmHandler.CleanUp() } func TestFrontMatterSplitWithArray(t *testing.T) { file := createTestFile(`[1,2,3] --- not a yaml: doc `) expectedYamlFm := "[1,2,3]\n" expectedContent := `--- not a yaml: doc ` fmHandler := NewFrontMatterHandler(file) err := fmHandler.Split() if err != nil { panic(err) } yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename()) test.AssertResult(t, expectedYamlFm, yamlFm) contentBytes, err := io.ReadAll(fmHandler.GetContentReader()) if err != nil { panic(err) } test.AssertResult(t, expectedContent, string(contentBytes)) tryRemoveTempFile(file) fmHandler.CleanUp() } ================================================ FILE: pkg/yqlib/goccy_yaml_test.go ================================================ package yqlib import ( "testing" "github.com/mikefarah/yq/v4/test" ) var goccyYamlFormatScenarios = []formatScenario{ { description: "basic - 3", skipDoc: true, input: "3", expected: "3\n", }, { description: "basic - 3.1", skipDoc: true, input: "3.1", expected: "3.1\n", }, { description: "basic - mike", skipDoc: true, input: "mike: 3", expected: "mike: 3\n", }, { description: "basic - map multiple entries", skipDoc: true, input: "mike: 3\nfred: 12\n", expected: "mike: 3\nfred: 12\n", }, { description: "basic - 3.1", skipDoc: true, input: "{\n mike: 3\n}", expected: "{mike: 3}\n", }, { description: "basic - tag with number", skipDoc: true, input: "mike: !!cat 3", expected: "mike: !!cat 3\n", }, { description: "basic - array of numbers", skipDoc: true, input: "- 3", expected: "- 3\n", }, { description: "basic - single line array", skipDoc: true, input: "[3]", expected: "[3]\n", }, { description: "basic - plain string", skipDoc: true, input: `a: meow`, expected: "a: meow\n", }, { description: "basic - double quoted string", skipDoc: true, input: `a: "meow"`, expected: "a: \"meow\"\n", }, { description: "basic - single quoted string", skipDoc: true, input: `a: 'meow'`, expected: "a: 'meow'\n", }, { description: "basic - string block", skipDoc: true, input: "a: |\n meow\n", expected: "a: |\n meow\n", }, { description: "basic - long string", skipDoc: true, input: "a: the cute cat wrote a long sentence that wasn't wrapped at all.\n", expected: "a: the cute cat wrote a long sentence that wasn't wrapped at all.\n", }, { description: "basic - string block", skipDoc: true, input: "a: |-\n meow\n", expected: "a: |-\n meow\n", }, { description: "basic - line comment", skipDoc: true, input: "a: meow # line comment\n", expected: "a: meow # line comment\n", }, // { // description: "basic - head comment", // skipDoc: true, // input: "# head comment\na: meow\n", // expected: "# head comment\na: meow\n", // go-yaml does this // }, // { // description: "basic - head and line comment", // skipDoc: true, // input: "# head comment\na: #line comment\n meow\n", // expected: "# head comment\na: meow #line comment\n", // go-yaml does this // }, { description: "basic - foot comment", skipDoc: true, input: "a: meow\n# foot comment\n", expected: "a: meow\n# foot comment\n", }, { description: "basic - foot comment", skipDoc: true, input: "a: meow\nb: woof\n# foot comment\n", expected: "a: meow\nb: woof\n# foot comment\n", }, { description: "basic - boolean", skipDoc: true, input: "true\n", expected: "true\n", }, { description: "basic - null", skipDoc: true, input: "a: null\n", expected: "a: null\n", }, { description: "basic - ~", skipDoc: true, input: "a: ~\n", expected: "a: ~\n", }, { description: "basic - ~", skipDoc: true, input: "null\n", expected: "null\n", }, { skipDoc: true, description: "blank value round trip", input: "test:", expected: "test:\n", }, { skipDoc: true, description: "trailing comment", input: "test: null\n# this comment will be removed", expected: "test: null\n# this comment will be removed\n", }, // { // description: "doc separator", // skipDoc: true, // input: "# hi\n---\na: cat\n---", // expected: "---\na: cat\n", // }, // { // description: "scalar with doc separator", // skipDoc: true, // input: "--- cat", // expected: "---\ncat\n", // }, { description: "scalar with doc separator", skipDoc: true, input: "---cat", expected: "---cat\n", }, { description: "basic - null", skipDoc: true, input: "null", expected: "null\n", }, { description: "basic - ~", skipDoc: true, input: "~", expected: "~\n", }, { description: "octal", skipDoc: true, input: "0o30", expression: "tag", expected: "!!int\n", }, { description: "basic - [null]", skipDoc: true, input: "[null]", expected: "[null]\n", }, { description: "multi document", skipDoc: true, input: "a: mike\n---\nb: remember", expected: "a: mike\n---\nb: remember\n", }, { description: "single doc anchor map", skipDoc: true, input: "a: &remember mike\nb: *remember", expected: "a: &remember mike\nb: *remember\n", }, { description: "explode doc anchor map", skipDoc: true, input: "a: &remember mike\nb: *remember", expression: "explode(.)", expected: "a: mike\nb: mike\n", }, { description: "multi document anchor map", skipDoc: true, input: "a: &remember mike\n---\nb: *remember", expression: "explode(.)", expected: "a: mike\n---\nb: mike\n", }, { description: "merge anchor", skipDoc: true, input: "a: &remember\n c: mike\nb:\n <<: *remember", // fine to have !!merge as that's what the current impl does expected: "a: &remember\n c: mike\nb:\n !!merge <<: *remember\n", }, { description: "custom tag", skipDoc: true, input: "a: !cat mike", expected: "a: !cat mike\n", }, { description: "basic - [~]", skipDoc: true, input: "[~]", expected: "[~]\n", }, { description: "basic - null map value", skipDoc: true, input: "a: null", expected: "a: null\n", }, { description: "basic - number", skipDoc: true, input: "3", expected: "3\n", }, { description: "basic - float", skipDoc: true, input: "3.1", expected: "3.1\n", }, { description: "basic - float", skipDoc: true, input: "[1, 2]", expected: "[1, 2]\n", }, } func testGoccyYamlScenario(t *testing.T, s formatScenario) { test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) } func TestGoccyYmlFormatScenarios(t *testing.T) { for _, tt := range goccyYamlFormatScenarios { testGoccyYamlScenario(t, tt) } } ================================================ FILE: pkg/yqlib/hcl.go ================================================ package yqlib type HclPreferences struct { ColorsEnabled bool } func NewDefaultHclPreferences() HclPreferences { return HclPreferences{ColorsEnabled: false} } func (p *HclPreferences) Copy() HclPreferences { return HclPreferences{ColorsEnabled: p.ColorsEnabled} } var ConfiguredHclPreferences = NewDefaultHclPreferences() ================================================ FILE: pkg/yqlib/hcl_test.go ================================================ //go:build !yq_nohcl package yqlib import ( "bufio" "bytes" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) var nestedExample = `service "http" "web_proxy" { listen_addr = "127.0.0.1:8080" }` var nestedExampleYaml = "service:\n http:\n web_proxy:\n listen_addr: \"127.0.0.1:8080\"\n" var multipleBlockLabelKeys = `service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ` var multipleBlockLabelKeysExpected = `service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ` var multipleBlockLabelKeysExpectedUpdate = `service "cat" { process "main" { command = ["/usr/local/bin/awesome-app", "server", "meow"] } process "management" { command = ["/usr/local/bin/awesome-app", "management"] } } ` var multipleBlockLabelKeysExpectedYaml = `service: cat: process: main: command: - "/usr/local/bin/awesome-app" - "server" management: command: - "/usr/local/bin/awesome-app" - "management" ` var simpleSample = `# Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message)` var simpleSampleExpected = `# Arithmetic with literals and application-provided variables sum = 1 + addend # String interpolation and templates message = "Hello, ${name}!" # Application-provided functions shouty_message = upper(message) ` var simpleSampleExpectedYaml = `# Arithmetic with literals and application-provided variables sum: 1 + addend # String interpolation and templates message: "Hello, ${name}!" # Application-provided functions shouty_message: upper(message) ` var hclFormatScenarios = []formatScenario{ { description: "Parse HCL", input: `io_mode = "async"`, expected: "io_mode: \"async\"\n", scenarioType: "decode", }, { description: "Simple decode, no quotes", skipDoc: true, input: `io_mode = async`, expected: "io_mode: async\n", scenarioType: "decode", }, { description: "Simple roundtrip, no quotes", skipDoc: true, input: `io_mode = async`, expected: "io_mode = async\n", scenarioType: "roundtrip", }, { description: "Nested decode", skipDoc: true, input: nestedExample, expected: nestedExampleYaml, scenarioType: "decode", }, { description: "Template decode", skipDoc: true, input: `message = "Hello, ${name}!"`, expected: "message: \"Hello, ${name}!\"\n", scenarioType: "decode", }, { description: "Roundtrip: with template", skipDoc: true, input: `message = "Hello, ${name}!"`, expected: "message = \"Hello, ${name}!\"\n", scenarioType: "roundtrip", }, { description: "Roundtrip: with function", skipDoc: true, input: `shouty_message = upper(message)`, expected: "shouty_message = upper(message)\n", scenarioType: "roundtrip", }, { description: "Roundtrip: with arithmetic", skipDoc: true, input: `sum = 1 + addend`, expected: "sum = 1 + addend\n", scenarioType: "roundtrip", }, { description: "Arithmetic decode", skipDoc: true, input: `sum = 1 + addend`, expected: "sum: 1 + addend\n", scenarioType: "decode", }, { description: "number attribute", skipDoc: true, input: `port = 8080`, expected: "port: 8080\n", scenarioType: "decode", }, { description: "float attribute", skipDoc: true, input: `pi = 3.14`, expected: "pi: 3.14\n", scenarioType: "decode", }, { description: "boolean attribute", skipDoc: true, input: `enabled = true`, expected: "enabled: true\n", scenarioType: "decode", }, { description: "object/map attribute", skipDoc: true, input: `obj = { a = 1, b = "two" }`, expected: "obj: {a: 1, b: \"two\"}\n", scenarioType: "decode", }, { description: "nested block", skipDoc: true, input: `server { port = 8080 }`, expected: "server:\n port: 8080\n", scenarioType: "decode", }, { description: "multiple attributes", skipDoc: true, input: "name = \"app\"\nversion = 1\nenabled = true", expected: "name: \"app\"\nversion: 1\nenabled: true\n", scenarioType: "decode", }, { description: "binary expression", skipDoc: true, input: `count = 0 - 42`, expected: "count: -42\n", scenarioType: "decode", }, { description: "negative number", skipDoc: true, input: `count = -42`, expected: "count: -42\n", scenarioType: "decode", }, { description: "scientific notation", skipDoc: true, input: `value = 1e-3`, expected: "value: 0.001\n", scenarioType: "decode", }, { description: "nested object", skipDoc: true, input: `config = { db = { host = "localhost", port = 5432 } }`, expected: "config: {db: {host: \"localhost\", port: 5432}}\n", scenarioType: "decode", }, { description: "mixed list", skipDoc: true, input: `values = [1, "two", true]`, expected: "values:\n - 1\n - \"two\"\n - true\n", scenarioType: "decode", }, { description: "Roundtrip: Sample Doc", input: multipleBlockLabelKeys, expected: multipleBlockLabelKeysExpected, scenarioType: "roundtrip", }, { description: "Roundtrip: With an update", input: multipleBlockLabelKeys, expression: `.service.cat.process.main.command += "meow"`, expected: multipleBlockLabelKeysExpectedUpdate, scenarioType: "roundtrip", }, { description: "Parse HCL: Sample Doc", input: multipleBlockLabelKeys, expected: multipleBlockLabelKeysExpectedYaml, scenarioType: "decode", }, { description: "block with labels", skipDoc: true, input: `resource "aws_instance" "example" { ami = "ami-12345" }`, expected: "resource:\n aws_instance:\n example:\n ami: \"ami-12345\"\n", scenarioType: "decode", }, { description: "block with labels roundtrip", skipDoc: true, input: `resource "aws_instance" "example" { ami = "ami-12345" }`, expected: "resource \"aws_instance\" \"example\" {\n ami = \"ami-12345\"\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip simple attribute", skipDoc: true, input: `io_mode = "async"`, expected: `io_mode = "async"` + "\n", scenarioType: "roundtrip", }, { description: "roundtrip number attribute", skipDoc: true, input: `port = 8080`, expected: "port = 8080\n", scenarioType: "roundtrip", }, { description: "roundtrip float attribute", skipDoc: true, input: `pi = 3.14`, expected: "pi = 3.14\n", scenarioType: "roundtrip", }, { description: "roundtrip boolean attribute", skipDoc: true, input: `enabled = true`, expected: "enabled = true\n", scenarioType: "roundtrip", }, { description: "roundtrip list of strings", skipDoc: true, input: `tags = ["a", "b"]`, expected: "tags = [\"a\", \"b\"]\n", scenarioType: "roundtrip", }, { description: "roundtrip object/map attribute", skipDoc: true, input: `obj = { a = 1, b = "two" }`, expected: "obj = {\n a = 1\n b = \"two\"\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip nested block", skipDoc: true, input: `server { port = 8080 }`, expected: "server {\n port = 8080\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip multiple attributes", skipDoc: true, input: "name = \"app\"\nversion = 1\nenabled = true", expected: "name = \"app\"\nversion = 1\nenabled = true\n", scenarioType: "roundtrip", }, { description: "Parse HCL: with comments", input: "# Configuration\nport = 8080 # server port", expected: "# Configuration\nport: 8080 # server port\n", scenarioType: "decode", }, { description: "Roundtrip: with comments", input: "# Configuration\nport = 8080", expected: "# Configuration\nport = 8080\n", scenarioType: "roundtrip", }, { description: "Roundtrip: extraction", skipDoc: true, input: simpleSample, expression: ".shouty_message", expected: "upper(message)\n", scenarioType: "roundtrip", }, { description: "Roundtrip: With templates, functions and arithmetic", input: simpleSample, expected: simpleSampleExpected, scenarioType: "roundtrip", }, { description: "roundtrip example", skipDoc: true, input: simpleSample, expected: simpleSampleExpectedYaml, scenarioType: "decode", }, { description: "Parse HCL: List of strings", skipDoc: true, input: `tags = ["a", "b"]`, expected: "tags:\n - \"a\"\n - \"b\"\n", scenarioType: "decode", }, { description: "roundtrip list of objects", skipDoc: true, input: `items = [{ name = "a", value = 1 }, { name = "b", value = 2 }]`, expected: "items = [{\n name = \"a\"\n value = 1\n }, {\n name = \"b\"\n value = 2\n}]\n", scenarioType: "roundtrip", }, { description: "roundtrip nested blocks with same name", skipDoc: true, input: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}", expected: "database \"primary\" {\n host = \"localhost\"\n port = 5432\n}\ndatabase \"replica\" {\n host = \"replica.local\"\n port = 5433\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip mixed nested structure", skipDoc: true, input: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}", expected: "servers \"web\" {\n addresses = [\"10.0.1.1\", \"10.0.1.2\"]\n port = 8080\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip null value", skipDoc: true, input: `value = null`, expected: "value = null\n", scenarioType: "roundtrip", }, { description: "roundtrip empty list", skipDoc: true, input: `items = []`, expected: "items = []\n", scenarioType: "roundtrip", }, { description: "roundtrip empty object", skipDoc: true, input: `config = {}`, expected: "config = {}\n", scenarioType: "roundtrip", }, { description: "Roundtrip: Separate blocks with same name.", input: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}", expected: "resource \"aws_instance\" \"web\" {\n ami = \"ami-12345\"\n}\nresource \"aws_instance\" \"db\" {\n ami = \"ami-67890\"\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip deeply nested structure", skipDoc: true, input: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}", expected: "app \"database\" \"primary\" \"connection\" {\n host = \"db.local\"\n port = 5432\n}\n", scenarioType: "roundtrip", }, { description: "roundtrip with leading comments", skipDoc: true, input: "# Main config\nenabled = true\nport = 8080", expected: "# Main config\nenabled = true\nport = 8080\n", scenarioType: "roundtrip", }, { description: "Multiple attributes with comments (comment safety with safe path separator)", skipDoc: true, input: "# Database config\ndb_host = \"localhost\"\n# Connection pool\ndb_pool = 10", expected: "# Database config\ndb_host = \"localhost\"\n# Connection pool\ndb_pool = 10\n", scenarioType: "roundtrip", }, { description: "Nested blocks with head comments", skipDoc: true, input: "service \"api\" {\n # Listen address\n listen = \"0.0.0.0:8080\"\n # TLS enabled\n tls = true\n}", expected: "service \"api\" {\n # Listen address\n listen = \"0.0.0.0:8080\"\n # TLS enabled\n tls = true\n}\n", scenarioType: "roundtrip", }, { description: "Multiple blocks with EncodeSeparate preservation", skipDoc: true, input: "resource \"aws_s3_bucket\" \"bucket1\" {\n bucket = \"my-bucket-1\"\n}\nresource \"aws_s3_bucket\" \"bucket2\" {\n bucket = \"my-bucket-2\"\n}", expected: "resource \"aws_s3_bucket\" \"bucket1\" {\n bucket = \"my-bucket-1\"\n}\nresource \"aws_s3_bucket\" \"bucket2\" {\n bucket = \"my-bucket-2\"\n}\n", scenarioType: "roundtrip", }, { description: "Blocks with same name handled separately", skipDoc: true, input: "server \"primary\" { port = 8080 }\nserver \"backup\" { port = 8081 }", expected: "server \"primary\" {\n port = 8080\n}\nserver \"backup\" {\n port = 8081\n}\n", scenarioType: "roundtrip", }, { description: "Block label with dot roundtrip (commentPathSep)", skipDoc: true, input: "service \"api.service\" {\n port = 8080\n}", expected: "service \"api.service\" {\n port = 8080\n}\n", scenarioType: "roundtrip", }, { description: "Nested template expression", skipDoc: true, input: `message = "User: ${username}, Role: ${user_role}"`, expected: "message = \"User: ${username}, Role: ${user_role}\"\n", scenarioType: "roundtrip", }, { description: "Empty object roundtrip", skipDoc: true, input: `obj = {}`, expected: "obj = {}\n", scenarioType: "roundtrip", }, { description: "Null value in block", skipDoc: true, input: `service { optional_field = null }`, expected: "service {\n optional_field = null\n}\n", scenarioType: "roundtrip", }, } func testHclScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "decode": result := mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)) test.AssertResultWithContext(t, s.expected, result, s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewHclDecoder(), NewHclEncoder(ConfiguredHclPreferences)), s.description) } } func documentHclScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "decode": documentHclDecodeScenario(w, s) case "roundtrip": documentHclRoundTripScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentHclDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.hcl file of:\n") writeOrPanic(w, fmt.Sprintf("```hcl\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if s.expression != "" { expression = fmt.Sprintf(" '%v'", s.expression) } writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy%v sample.hcl\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewHclDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentHclRoundTripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.hcl file of:\n") writeOrPanic(w, fmt.Sprintf("```hcl\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if s.expression != "" { expression = fmt.Sprintf(" '%v'", s.expression) } writeOrPanic(w, fmt.Sprintf("```bash\nyq%v sample.hcl\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```hcl\n%v```\n\n", mustProcessFormatScenario(s, NewHclDecoder(), NewHclEncoder(ConfiguredHclPreferences)))) } func TestHclEncoderPrintDocumentSeparator(t *testing.T) { encoder := NewHclEncoder(ConfiguredHclPreferences) var buf bytes.Buffer writer := bufio.NewWriter(&buf) err := encoder.PrintDocumentSeparator(writer) writer.Flush() test.AssertResult(t, nil, err) test.AssertResult(t, "", buf.String()) } func TestHclEncoderPrintLeadingContent(t *testing.T) { encoder := NewHclEncoder(ConfiguredHclPreferences) var buf bytes.Buffer writer := bufio.NewWriter(&buf) err := encoder.PrintLeadingContent(writer, "some content") writer.Flush() test.AssertResult(t, nil, err) test.AssertResult(t, "", buf.String()) } func TestHclEncoderCanHandleAliases(t *testing.T) { encoder := NewHclEncoder(ConfiguredHclPreferences) test.AssertResult(t, false, encoder.CanHandleAliases()) } func TestHclFormatScenarios(t *testing.T) { for _, tt := range hclFormatScenarios { testHclScenario(t, tt) } genericScenarios := make([]interface{}, len(hclFormatScenarios)) for i, s := range hclFormatScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "hcl", genericScenarios, documentHclScenario) } ================================================ FILE: pkg/yqlib/ini.go ================================================ package yqlib type INIPreferences struct { ColorsEnabled bool } func NewDefaultINIPreferences() INIPreferences { return INIPreferences{ ColorsEnabled: false, } } func (p *INIPreferences) Copy() INIPreferences { return INIPreferences{ ColorsEnabled: p.ColorsEnabled, } } var ConfiguredINIPreferences = NewDefaultINIPreferences() ================================================ FILE: pkg/yqlib/ini_test.go ================================================ //go:build !yq_noini package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const simpleINIInput = `[section] key = value ` const expectedSimpleINIOutput = `[section] key = value ` const expectedSimpleINIYaml = `section: key: value ` var iniScenarios = []formatScenario{ { description: "Parse INI: simple", input: simpleINIInput, scenarioType: "decode", expected: expectedSimpleINIYaml, }, { description: "Encode INI: simple", input: `section: {key: value}`, expected: expectedSimpleINIOutput, scenarioType: "encode", }, { description: "Roundtrip INI: simple", input: simpleINIInput, expected: expectedSimpleINIOutput, scenarioType: "roundtrip", }, { description: "bad ini", input: `[section\nkey = value`, expectedError: `bad file 'sample.yml': failed to parse INI content: unclosed section: [section\nkey = value`, scenarioType: "decode-error", }, } func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.ini file of:\n") writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini -o=ini '%v' sample.ini\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -p=ini -o=ini sample.ini\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder()))) } func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.ini file of:\n") writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=ini '%v' sample.ini\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -p=ini sample.ini\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func testINIScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder()), s.description) case "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder()), s.description) case "decode-error": result, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder()) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentINIScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "encode": documentINIEncodeScenario(w, s) case "decode": documentDecodeINIScenario(w, s) case "roundtrip": documentRoundtripINIScenario(w, s) case "decode-error": documentDecodeErrorINIScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentINIEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=ini '%v' sample.yml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder()))) } func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.ini file of:\n") writeOrPanic(w, fmt.Sprintf("```ini\n%v\n```\n", s.input)) writeOrPanic(w, "then an error is expected:\n") writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError)) } func TestINIScenarios(t *testing.T) { for _, tt := range iniScenarios { testINIScenario(t, tt) } genericScenarios := make([]interface{}, len(iniScenarios)) for i, s := range iniScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario) } ================================================ FILE: pkg/yqlib/json.go ================================================ package yqlib type JsonPreferences struct { Indent int ColorsEnabled bool UnwrapScalar bool } func NewDefaultJsonPreferences() JsonPreferences { return JsonPreferences{ Indent: 2, ColorsEnabled: true, UnwrapScalar: true, } } func (p *JsonPreferences) Copy() JsonPreferences { return JsonPreferences{ Indent: p.Indent, ColorsEnabled: p.ColorsEnabled, UnwrapScalar: p.UnwrapScalar, } } var ConfiguredJSONPreferences = NewDefaultJsonPreferences() ================================================ FILE: pkg/yqlib/json_test.go ================================================ //go:build !yq_nojson package yqlib import ( "bufio" "bytes" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const complexExpectYaml = `a: Easy! as one two three b: c: 2 d: - 3 - 4 ` const sampleNdJson = `{"this": "is a multidoc json file"} {"each": ["line is a valid json document"]} {"a number": 4} ` const sampleNdJsonKey = `{"a": "first", "b": "next", "ab": "last"}` const expectedJsonKeysInOrder = `a: first b: next ab: last ` const expectedNdJsonYaml = `this: is a multidoc json file --- each: - line is a valid json document --- a number: 4 ` const expectedRoundTripSampleNdJson = `{"this":"is a multidoc json file"} {"each":["line is a valid json document"]} {"a number":4} ` const expectedUpdatedMultilineJson = `{"this":"is a multidoc json file"} {"each":["line is a valid json document","cool"]} {"a number":4} ` const sampleMultiLineJson = `{ "this": "is a multidoc json file" } { "it": [ "has", "consecutive", "json documents" ] } { "a number": 4 } ` const roundTripMultiLineJson = `{ "this": "is a multidoc json file" } { "it": [ "has", "consecutive", "json documents" ] } { "a number": 4 } ` var jsonScenarios = []formatScenario{ { description: "array empty", skipDoc: true, input: "[]", scenarioType: "roundtrip-ndjson", indent: 0, expected: "[]\n", }, { description: "array has scalar", skipDoc: true, input: "[3]", scenarioType: "roundtrip-ndjson", indent: 0, expected: "[3]\n", }, { description: "array has object", skipDoc: true, input: `[{"x": 3}]`, scenarioType: "roundtrip-ndjson", indent: 0, expected: "[{\"x\":3}]\n", }, { description: "array null", skipDoc: true, input: "[null]", scenarioType: "roundtrip-ndjson", indent: 0, expected: "[null]\n", }, { description: "set tags", skipDoc: true, input: "[{}]", expression: `[.. | type]`, scenarioType: "roundtrip-ndjson", indent: 0, expected: "[\"!!seq\",\"!!map\"]\n", }, { description: "Parse json: simple", subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", input: `{"cat": "meow"}`, scenarioType: "decode-ndjson", expected: "cat: meow\n", }, { skipDoc: true, description: "Parse json: simple: key", input: `{"cat": "meow"}`, expression: ".cat | key", expected: "\"cat\"\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse json: simple: parent", input: `{"cat": "meow"}`, expression: ".cat | parent", expected: "{\"cat\":\"meow\"}\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse json: simple: path", input: `{"cat": "meow"}`, expression: ".cat | path", expected: "[\"cat\"]\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse json: deeper: path", input: `{"cat": {"noises": "meow"}}`, expression: ".cat.noises | path", expected: "[\"cat\",\"noises\"]\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse json: array path", input: `{"cat": {"noises": ["meow"]}}`, expression: ".cat.noises[0] | path", expected: "[\"cat\",\"noises\",0]\n", scenarioType: "decode", }, { description: "bad json", skipDoc: true, input: `{"a": 1 b": 2}`, expectedError: `bad file 'sample.yml': json: string of object unexpected end of JSON input`, scenarioType: "decode-error", }, { description: "Parse json: complex", subdescription: "JSON is a subset of yaml, so all you need to do is prettify the output", input: `{"a":"Easy! as one two three","b":{"c":2,"d":[3,4]}}`, expected: complexExpectYaml, scenarioType: "decode-ndjson", }, { description: "Encode json: simple", input: `cat: meow`, indent: 2, expected: "{\n \"cat\": \"meow\"\n}\n", scenarioType: "encode", }, { description: "Encode json: simple - in one line", input: `cat: meow # this is a comment, and it will be dropped.`, indent: 0, expected: "{\"cat\":\"meow\"}\n", scenarioType: "encode", }, { description: "Encode json: comments", input: `cat: meow # this is a comment, and it will be dropped.`, indent: 2, expected: "{\n \"cat\": \"meow\"\n}\n", scenarioType: "encode", }, { description: "Encode json: anchors", subdescription: "Anchors are dereferenced", input: "cat: &ref meow\nanotherCat: *ref", indent: 2, expected: "{\n \"cat\": \"meow\",\n \"anotherCat\": \"meow\"\n}\n", scenarioType: "encode", }, { description: "Encode json: multiple results", subdescription: "Each matching node is converted into a json doc. This is best used with 0 indent (json document per line)", input: `things: [{stuff: cool}, {whatever: cat}]`, expression: `.things[]`, indent: 0, expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n", scenarioType: "encode", }, { description: "Roundtrip JSON Lines / NDJSON", input: sampleNdJson, expected: expectedRoundTripSampleNdJson, scenarioType: "roundtrip-ndjson", indent: 0, }, { description: "Roundtrip multi-document JSON", subdescription: "The parser can also handle multiple multi-line json documents in a single file (despite this not being in the JSON Lines / NDJSON spec). Typically you would have one entire JSON document per line, but the parser also supports multiple multi-line json documents", input: sampleMultiLineJson, expected: roundTripMultiLineJson, scenarioType: "roundtrip-multi", }, { description: "Update a specific document in a multi-document json", subdescription: "Documents are indexed by the `documentIndex` or `di` operator.", input: sampleNdJson, expected: expectedUpdatedMultilineJson, expression: `(select(di == 1) | .each ) += "cool"`, scenarioType: "roundtrip-ndjson", }, { description: "Find and update a specific document in a multi-document json", subdescription: "Use expressions as you normally would.", input: sampleNdJson, expected: expectedUpdatedMultilineJson, expression: `(select(has("each")) | .each ) += "cool"`, scenarioType: "roundtrip-ndjson", }, { description: "Decode JSON Lines / NDJSON", input: sampleNdJson, expected: expectedNdJsonYaml, scenarioType: "decode-ndjson", }, { description: "Decode JSON Lines / NDJSON, maintain key order", skipDoc: true, input: sampleNdJsonKey, expected: expectedJsonKeysInOrder, scenarioType: "decode-ndjson", }, { description: "numbers", skipDoc: true, input: "[3, 3.0, 3.1, -1, 999999, 1000000, 1000001, 1.1]", expected: "- 3\n- 3\n- 3.1\n- -1\n- 999999\n- 1000000\n- 1000001\n- 1.1\n", scenarioType: "decode-ndjson", }, { description: "number single", skipDoc: true, input: "3", expected: "3\n", scenarioType: "decode-ndjson", }, { description: "empty string", skipDoc: true, input: `""`, expected: "\n", scenarioType: "decode-ndjson", }, { description: "strings", skipDoc: true, input: `["", "cat"]`, expected: "- \"\"\n- cat\n", scenarioType: "decode-ndjson", }, { description: "null", skipDoc: true, input: `null`, expected: "null\n", scenarioType: "decode-ndjson", }, { description: "booleans", skipDoc: true, input: `[true, false]`, expected: "- true\n- false\n", scenarioType: "decode-ndjson", }, } func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.json file of:\n") writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v '%v' sample.json\n```\n", indent, expression)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json -o=json -I=%v sample.json\n```\n", indent)) } writeOrPanic(w, "will output\n") prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = indent prefs.UnwrapScalar = false writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)))) } func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.json file of:\n") writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=json '%v' sample.json\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -p=json sample.json\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func decodeJSON(t *testing.T, jsonString string) *CandidateNode { docs, err := readDocument(jsonString, "sample.json", 0) if err != nil { t.Error(err) return nil } exp, err := getExpressionParser().ParseExpression(PrettyPrintExp) if err != nil { t.Error(err) return nil } context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: docs}, exp) if err != nil { t.Error(err) return nil } return context.MatchingNodes.Front().Value.(*CandidateNode) } func testJSONScenario(t *testing.T, s formatScenario) { prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = s.indent prefs.UnwrapScalar = false switch s.scenarioType { case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs)), s.description) case "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description) case "decode-ndjson": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "roundtrip-ndjson": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description) case "roundtrip-multi": prefs.Indent = 2 test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)), s.description) case "decode-error": result, err := processFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs)) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.json file of:\n") writeOrPanic(w, fmt.Sprintf("```json\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq -P '.' sample.json\n```\n") writeOrPanic(w, "will output\n") var output bytes.Buffer printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true) node := decodeJSON(t, s.input) err := printer.PrintResults(node.AsList()) if err != nil { t.Error(err) return } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String())) } func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "": documentJSONDecodeScenario(t, w, s) case "encode": documentJSONEncodeScenario(w, s) case "decode-ndjson": documentDecodeNdJsonScenario(w, s) case "roundtrip-ndjson": documentRoundtripNdJsonScenario(w, s, 0) case "roundtrip-multi": documentRoundtripNdJsonScenario(w, s, 2) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } if s.indent == 2 { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json '%v' sample.yml\n```\n", expression)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=json -I=%v '%v' sample.yml\n```\n", s.indent, expression)) } writeOrPanic(w, "will output\n") prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = s.indent prefs.UnwrapScalar = false writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewJSONEncoder(prefs)))) } func TestJSONScenarios(t *testing.T) { for _, tt := range jsonScenarios { testJSONScenario(t, tt) } genericScenarios := make([]interface{}, len(jsonScenarios)) for i, s := range jsonScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "convert", genericScenarios, documentJSONScenario) } ================================================ FILE: pkg/yqlib/kyaml.go ================================================ //go:build !yq_nokyaml package yqlib type KYamlPreferences struct { Indent int ColorsEnabled bool PrintDocSeparators bool UnwrapScalar bool } func NewDefaultKYamlPreferences() KYamlPreferences { return KYamlPreferences{ Indent: 2, ColorsEnabled: false, PrintDocSeparators: true, UnwrapScalar: true, } } func (p *KYamlPreferences) Copy() KYamlPreferences { return KYamlPreferences{ Indent: p.Indent, ColorsEnabled: p.ColorsEnabled, PrintDocSeparators: p.PrintDocSeparators, UnwrapScalar: p.UnwrapScalar, } } var ConfiguredKYamlPreferences = NewDefaultKYamlPreferences() ================================================ FILE: pkg/yqlib/kyaml_test.go ================================================ //go:build !yq_nokyaml package yqlib import ( "bufio" "bytes" "fmt" "regexp" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) var ansiRe = regexp.MustCompile(`\x1b\[[0-9;]*m`) func stripANSI(s string) string { return ansiRe.ReplaceAllString(s, "") } var kyamlFormatScenarios = []formatScenario{ { description: "Encode kyaml: plain string scalar", subdescription: "Strings are always double-quoted in KYaml output.", scenarioType: "encode", indent: 2, input: "cat\n", expected: "\"cat\"\n", }, { description: "encode plain int scalar", scenarioType: "encode", indent: 2, input: "12\n", expected: "12\n", skipDoc: true, }, { description: "encode plain bool scalar", scenarioType: "encode", indent: 2, input: "true\n", expected: "true\n", skipDoc: true, }, { description: "encode plain null scalar", scenarioType: "encode", indent: 2, input: "null\n", expected: "null\n", skipDoc: true, }, { description: "encode flow mapping and sequence", scenarioType: "encode", indent: 2, input: "a: b\nc:\n - d\n", expected: "{\n" + " a: \"b\",\n" + " c: [\n" + " \"d\",\n" + " ],\n" + "}\n", }, { description: "encode non-string scalars", scenarioType: "encode", indent: 2, input: "a: 12\n" + "b: true\n" + "c: null\n" + "d: \"true\"\n", expected: "{\n" + " a: 12,\n" + " b: true,\n" + " c: null,\n" + " d: \"true\",\n" + "}\n", }, { description: "quote non-identifier keys", scenarioType: "encode", indent: 2, input: "\"1a\": b\n\"has space\": c\n", expected: "{\n" + " \"1a\": \"b\",\n" + " \"has space\": \"c\",\n" + "}\n", }, { description: "escape quoted strings", scenarioType: "encode", indent: 2, input: "a: \"line1\\nline2\\t\\\"q\\\"\"\n", expected: "{\n" + " a: \"line1\\nline2\\t\\\"q\\\"\",\n" + "}\n", }, { description: "preserve comments when encoding", scenarioType: "encode", indent: 2, input: "# leading\n" + "a: 1 # a line\n" + "# head b\n" + "b: 2\n" + "c:\n" + " # head d\n" + " - d # d line\n" + " - e\n" + "# trailing\n", expected: "# leading\n" + "{\n" + " a: 1, # a line\n" + " # head b\n" + " b: 2,\n" + " c: [\n" + " # head d\n" + " \"d\", # d line\n" + " \"e\",\n" + " ],\n" + " # trailing\n" + "}\n", }, { description: "Encode kyaml: anchors and aliases", subdescription: "KYaml output does not support anchors/aliases; they are expanded to concrete values.", scenarioType: "encode", indent: 2, input: "base: &base\n" + " a: b\n" + "copy: *base\n", expected: "{\n" + " base: {\n" + " a: \"b\",\n" + " },\n" + " copy: {\n" + " a: \"b\",\n" + " },\n" + "}\n", }, { description: "Encode kyaml: yaml to kyaml shows formatting differences", subdescription: "KYaml uses flow-style collections (braces/brackets) and explicit commas.", scenarioType: "encode", indent: 2, input: "person:\n" + " name: John\n" + " pets:\n" + " - cat\n" + " - dog\n", expected: "{\n" + " person: {\n" + " name: \"John\",\n" + " pets: [\n" + " \"cat\",\n" + " \"dog\",\n" + " ],\n" + " },\n" + "}\n", }, { description: "Encode kyaml: nested lists of objects", subdescription: "Lists and objects can be nested arbitrarily; KYaml always uses flow-style collections.", scenarioType: "encode", indent: 2, input: "- name: a\n" + " items:\n" + " - id: 1\n" + " tags:\n" + " - k: x\n" + " v: y\n" + " - k: x2\n" + " v: y2\n" + " - id: 2\n" + " tags:\n" + " - k: z\n" + " v: w\n", expected: "[\n" + " {\n" + " name: \"a\",\n" + " items: [\n" + " {\n" + " id: 1,\n" + " tags: [\n" + " {\n" + " k: \"x\",\n" + " v: \"y\",\n" + " },\n" + " {\n" + " k: \"x2\",\n" + " v: \"y2\",\n" + " },\n" + " ],\n" + " },\n" + " {\n" + " id: 2,\n" + " tags: [\n" + " {\n" + " k: \"z\",\n" + " v: \"w\",\n" + " },\n" + " ],\n" + " },\n" + " ],\n" + " },\n" + "]\n", }, } func testKYamlScenario(t *testing.T, s formatScenario) { prefs := ConfiguredKYamlPreferences.Copy() prefs.Indent = s.indent prefs.UnwrapScalar = false switch s.scenarioType { case "encode": test.AssertResultWithContext( t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewKYamlEncoder(prefs)), s.description, ) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentKYamlScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "encode": documentKYamlEncodeScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentKYamlEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } if s.indent == 2 { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=kyaml '%v' sample.yml\n```\n", expression)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=kyaml -I=%v '%v' sample.yml\n```\n", s.indent, expression)) } writeOrPanic(w, "will output\n") prefs := ConfiguredKYamlPreferences.Copy() prefs.Indent = s.indent prefs.UnwrapScalar = false writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewKYamlEncoder(prefs)))) } func TestKYamlFormatScenarios(t *testing.T) { for _, s := range kyamlFormatScenarios { testKYamlScenario(t, s) } genericScenarios := make([]interface{}, len(kyamlFormatScenarios)) for i, s := range kyamlFormatScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "kyaml", genericScenarios, documentKYamlScenario) } func TestKYamlEncoderPrintDocumentSeparator(t *testing.T) { t.Run("enabled", func(t *testing.T) { prefs := NewDefaultKYamlPreferences() prefs.PrintDocSeparators = true var buf bytes.Buffer err := NewKYamlEncoder(prefs).PrintDocumentSeparator(&buf) if err != nil { t.Fatal(err) } if buf.String() != "---\n" { t.Fatalf("expected doc separator, got %q", buf.String()) } }) t.Run("disabled", func(t *testing.T) { prefs := NewDefaultKYamlPreferences() prefs.PrintDocSeparators = false var buf bytes.Buffer err := NewKYamlEncoder(prefs).PrintDocumentSeparator(&buf) if err != nil { t.Fatal(err) } if buf.String() != "" { t.Fatalf("expected no output, got %q", buf.String()) } }) } func TestKYamlEncoderEncodeUnwrapScalar(t *testing.T) { prefs := NewDefaultKYamlPreferences() prefs.UnwrapScalar = true var buf bytes.Buffer err := NewKYamlEncoder(prefs).Encode(&buf, &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "cat", }) if err != nil { t.Fatal(err) } if buf.String() != "cat\n" { t.Fatalf("expected unwrapped scalar, got %q", buf.String()) } } func TestKYamlEncoderEncodeColorsEnabled(t *testing.T) { prefs := NewDefaultKYamlPreferences() prefs.UnwrapScalar = false prefs.ColorsEnabled = true var buf bytes.Buffer err := NewKYamlEncoder(prefs).Encode(&buf, &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{ {Kind: ScalarNode, Tag: "!!str", Value: "a"}, {Kind: ScalarNode, Tag: "!!str", Value: "b"}, }, }) if err != nil { t.Fatal(err) } out := stripANSI(buf.String()) if !strings.Contains(out, "a:") || !strings.Contains(out, "\"b\"") { t.Fatalf("expected colourised output to contain rendered tokens, got %q", out) } } func TestKYamlEncoderWriteNodeAliasAndUnknown(t *testing.T) { ke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder) t.Run("alias_nil", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{Kind: AliasNode}, 0) if err != nil { t.Fatal(err) } if buf.String() != "null" { t.Fatalf("expected null for nil alias, got %q", buf.String()) } }) t.Run("alias_value", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{ Kind: AliasNode, Alias: &CandidateNode{Kind: ScalarNode, Tag: "!!int", Value: "12"}, }, 0) if err != nil { t.Fatal(err) } if buf.String() != "12" { t.Fatalf("expected dereferenced alias value, got %q", buf.String()) } }) t.Run("unknown_kind", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{Kind: Kind(12345)}, 0) if err != nil { t.Fatal(err) } if buf.String() != "null" { t.Fatalf("expected null for unknown kind, got %q", buf.String()) } }) } func TestKYamlEncoderEmptyCollections(t *testing.T) { ke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder) t.Run("empty_mapping", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{Kind: MappingNode}, 0) if err != nil { t.Fatal(err) } if buf.String() != "{}" { t.Fatalf("expected empty mapping, got %q", buf.String()) } }) t.Run("empty_sequence", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{Kind: SequenceNode}, 0) if err != nil { t.Fatal(err) } if buf.String() != "[]" { t.Fatalf("expected empty sequence, got %q", buf.String()) } }) } func TestKYamlEncoderScalarFallbackAndEscaping(t *testing.T) { ke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder) t.Run("unknown_tag_falls_back_to_string", func(t *testing.T) { var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{Kind: ScalarNode, Tag: "!!timestamp", Value: "2020-01-01T00:00:00Z"}, 0) if err != nil { t.Fatal(err) } if buf.String() != "\"2020-01-01T00:00:00Z\"" { t.Fatalf("expected quoted fallback, got %q", buf.String()) } }) t.Run("escape_double_quoted", func(t *testing.T) { got := escapeDoubleQuotedString("a\\b\"c\n\r\t" + string(rune(0x01))) want := "a\\\\b\\\"c\\n\\r\\t\\u0001" if got != want { t.Fatalf("expected %q, got %q", want, got) } }) t.Run("valid_bare_key", func(t *testing.T) { if isValidKYamlBareKey("") { t.Fatalf("expected empty string to be invalid") } if isValidKYamlBareKey("1a") { t.Fatalf("expected leading digit to be invalid") } if !isValidKYamlBareKey("a_b-2") { t.Fatalf("expected identifier-like key to be valid") } }) } func TestKYamlEncoderCommentsInMapping(t *testing.T) { prefs := NewDefaultKYamlPreferences() prefs.UnwrapScalar = false ke := NewKYamlEncoder(prefs).(*kyamlEncoder) var buf bytes.Buffer err := ke.writeNode(&buf, &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{ { Kind: ScalarNode, Tag: "!!str", Value: "a", HeadComment: "key head", LineComment: "key line", FootComment: "key foot", }, { Kind: ScalarNode, Tag: "!!str", Value: "b", HeadComment: "value head", }, }, }, 0) if err != nil { t.Fatal(err) } out := buf.String() if !strings.Contains(out, "# key head\n") { t.Fatalf("expected key head comment, got %q", out) } if !strings.Contains(out, "# value head\n") { t.Fatalf("expected value head comment, got %q", out) } if !strings.Contains(out, ", # key line\n") { t.Fatalf("expected inline key comment fallback, got %q", out) } if !strings.Contains(out, "# key foot\n") { t.Fatalf("expected foot comment fallback, got %q", out) } } func TestKYamlEncoderCommentBlockAndInlineComment(t *testing.T) { ke := NewKYamlEncoder(NewDefaultKYamlPreferences()).(*kyamlEncoder) t.Run("comment_block_prefixing_and_crlf", func(t *testing.T) { var buf bytes.Buffer err := ke.writeCommentBlock(&buf, "line1\r\n\r\n# already\r\nline2", 2) if err != nil { t.Fatal(err) } want := " # line1\n # already\n # line2\n" if buf.String() != want { t.Fatalf("expected %q, got %q", want, buf.String()) } }) t.Run("inline_comment_prefix_and_first_line_only", func(t *testing.T) { var buf bytes.Buffer err := ke.writeInlineComment(&buf, "hello\r\nsecond line") if err != nil { t.Fatal(err) } if buf.String() != " # hello" { t.Fatalf("expected %q, got %q", " # hello", buf.String()) } }) t.Run("inline_comment_already_prefixed", func(t *testing.T) { var buf bytes.Buffer err := ke.writeInlineComment(&buf, "# hello") if err != nil { t.Fatal(err) } if buf.String() != " # hello" { t.Fatalf("expected %q, got %q", " # hello", buf.String()) } }) } ================================================ FILE: pkg/yqlib/lexer.go ================================================ package yqlib import ( "fmt" "regexp" ) type expressionTokeniser interface { Tokenise(expression string) ([]*token, error) } type tokenType uint32 const ( operationToken = 1 << iota openBracket closeBracket openCollect closeCollect openCollectObject closeCollectObject traverseArrayCollect ) type token struct { TokenType tokenType Operation *Operation AssignOperation *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat Match string } func (t *token) toString(detail bool) string { switch t.TokenType { case operationToken: if detail { return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence) } return t.Operation.toString() case openBracket: return "(" case closeBracket: return ")" case openCollect: return "[" case closeCollect: return "]" case openCollectObject: return "{" case closeCollectObject: return "}" case traverseArrayCollect: return ".[" } return "NFI" } func unwrap(value string) string { return value[1 : len(value)-1] } func extractNumberParameter(value string) (int, error) { parameterParser := regexp.MustCompile(`.*\((-?[0-9]+)\)`) matches := parameterParser.FindStringSubmatch(value) var indent, errParsingInt = parseInt(matches[1]) if errParsingInt != nil { return 0, errParsingInt } return indent, nil } func hasOptionParameter(value string, option string) bool { parameterParser := regexp.MustCompile(`.*\([^\)]*\)`) matches := parameterParser.FindStringSubmatch(value) if len(matches) == 0 { return false } parameterString := matches[0] optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option)) return len(optionParser.FindStringSubmatch(parameterString)) > 0 } func postProcessTokens(tokens []*token) []*token { var postProcessedTokens = make([]*token, 0) skipNextToken := false for index := range tokens { if skipNextToken { skipNextToken = false } else { postProcessedTokens, skipNextToken = handleToken(tokens, index, postProcessedTokens) } } return postProcessedTokens } func tokenIsOpType(token *token, opType *operationType) bool { return token.TokenType == operationToken && token.Operation.OperationType == opType } func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) { skipNextToken = false currentToken := tokens[index] log.Debug("processing %v", currentToken.toString(true)) if currentToken.TokenType == traverseArrayCollect { // `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator //need to put a traverse array then a collect currentToken // do this by adding traverse then converting currentToken to collect log.Debug("adding self") op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) log.Debug("adding traverse array") op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) currentToken = &token{TokenType: openCollect} } if tokenIsOpType(currentToken, createMapOpType) { log.Debugf("tokenIsOpType: createMapOpType") // check the previous token is '[', means we are slice, but dont have a first number if index > 0 && tokens[index-1].TokenType == traverseArrayCollect { log.Debugf("previous token is : traverseArrayOpType") // need to put the number 0 before this token, as that is implied postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")}) } } if index != len(tokens)-1 && currentToken.AssignOperation != nil && tokenIsOpType(tokens[index+1], assignOpType) { log.Debug("its an update assign") currentToken.Operation = currentToken.AssignOperation currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign skipNextToken = true } log.Debug("adding token to the fixed list") postProcessedTokens = append(postProcessedTokens, currentToken) if tokenIsOpType(currentToken, createMapOpType) { log.Debugf("tokenIsOpType: createMapOpType") // check the next token is ']', means we are slice, but dont have a second number if index != len(tokens)-1 && tokens[index+1].TokenType == closeCollect { log.Debugf("next token is : closeCollect") // need to put the number 0 before this token, as that is implied lengthOp := &Operation{OperationType: lengthOpType} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: lengthOp}) } } if index != len(tokens)-1 && ((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) || (currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) { log.Debug("adding empty") op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } if index != len(tokens)-1 && currentToken.CheckForPostTraverse && (tokenIsOpType(tokens[index+1], traversePathOpType) || (tokens[index+1].TokenType == traverseArrayCollect)) { log.Debug("adding pipe because the next thing is traverse") op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == openCollect { log.Debug("adding traverseArray because next is opencollect") op := &Operation{OperationType: traverseArrayOpType} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } return postProcessedTokens, skipNextToken } ================================================ FILE: pkg/yqlib/lexer_participle.go ================================================ package yqlib import ( "strconv" "strings" "github.com/alecthomas/participle/v2/lexer" ) var participleYqRules = []*participleYqRule{ {"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0}, {"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0}, {"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0}, {"OpenBracket", `\(`, literalToken(openBracket, false), 0}, {"CloseBracket", `\)`, literalToken(closeBracket, true), 0}, {"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0}, {"OpenCollect", `\[`, literalToken(openCollect, false), 0}, {"CloseCollect", `\]\??`, literalToken(closeCollect, true), 0}, {"OpenCollectObject", `\{`, literalToken(openCollectObject, false), 0}, {"CloseCollectObject", `\}`, literalToken(closeCollectObject, true), 0}, {"RecursiveDecentIncludingKeys", `\.\.\.`, recursiveDecentOpToken(true), 0}, {"RecursiveDecent", `\.\.`, recursiveDecentOpToken(false), 0}, {"GetVariable", `\$[a-zA-Z_\-0-9]+`, getVariableOpToken(), 0}, {"AssignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0}, {"AssignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0}, {"CreateMap", `:\s*`, opToken(createMapOpType), 0}, simpleOp("length", lengthOpType), simpleOp("line", lineOpType), simpleOp("column", columnOpType), simpleOp("eval", evalOpType), simpleOp("to_?number", toNumberOpType), {"MapValues", `map_?values`, opToken(mapValuesOpType), 0}, simpleOp("map", mapOpType), simpleOp("filter", filterOpType), simpleOp("pick", pickOpType), simpleOp("omit", omitOpType), {"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0}, {"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0}, simpleOp("format_datetime", formatDateTimeOpType), simpleOp("now", nowOpType), simpleOp("tz", tzOpType), simpleOp("from_?unix", fromUnixOpType), simpleOp("to_?unix", toUnixOpType), simpleOp("with_dtf", withDtFormatOpType), simpleOp("error", errorOpType), simpleOp("shuffle", shuffleOpType), simpleOp("sortKeys", sortKeysOpType), simpleOp("sort_?keys", sortKeysOpType), {"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0}, {"Root", "root", expressionOpToken(`parent(-1)`), 0}, {"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlFormat), 0}, {"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLFormat), 0}, {"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONFormat), 0}, {"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlFormat), 0}, {"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlFormat, 2), 0}, {"JSONEncode", `to_?json`, encodeWithIndent(JSONFormat, 2), 0}, {"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONFormat, 0), 0}, {"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesFormat), 0}, {"PropsEncode", `to_?props|@props`, encodeWithIndent(PropertiesFormat, 2), 0}, {"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLFormat), 0}, {"XMLEncode", `to_?xml`, encodeWithIndent(XMLFormat, 2), 0}, {"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLFormat, 0), 0}, {"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVFormat), 0}, {"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVFormat, 0), 0}, {"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVFormat), 0}, {"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVFormat, 0), 0}, {"Base64d", `@base64d`, decodeOp(Base64Format), 0}, {"Base64", `@base64`, encodeWithIndent(Base64Format, 0), 0}, {"Urid", `@urid`, decodeOp(UriFormat), 0}, {"Uri", `@uri`, encodeWithIndent(UriFormat, 0), 0}, {"SH", `@sh`, encodeWithIndent(ShFormat, 0), 0}, {"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences)), 0}, {"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder()), 0}, {"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder()), 0}, simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), simpleOp("has", hasOpType), simpleOp("unique_?by", uniqueByOpType), simpleOp("unique", uniqueOpType), simpleOp("group_?by", groupByOpType), simpleOp("explode", explodeOpType), simpleOp("or", orOpType), simpleOp("and", andOpType), simpleOp("not", notOpType), simpleOp("ireduce", reduceOpType), simpleOp("join", joinStringOpType), simpleOp("sub", subStringOpType), simpleOp("match", matchOpType), simpleOp("capture", captureOpType), simpleOp("test", testOpType), simpleOp("sort_?by", sortByOpType), simpleOp("sort", sortOpType), simpleOp("first", firstOpType), simpleOp("reverse", reverseOpType), simpleOp("any_c", anyConditionOpType), simpleOp("any", anyOpType), simpleOp("all_c", allConditionOpType), simpleOp("all", allOpType), simpleOp("contains", containsOpType), simpleOp("split", splitStringOpType), simpleOp("parents", getParentsOpType), {"ParentWithLevel", `parent\(-?[0-9]+\)`, parentWithLevel(), 0}, {"ParentWithDefaultLevel", `parent`, parentWithDefaultLevel(), 0}, simpleOp("keys", keysOpType), simpleOp("key", getKeyOpType), simpleOp("is_?key", isKeyOpType), simpleOp("file_?name|fileName", getFilenameOpType), simpleOp("file_?index|fileIndex|fi", getFileIndexOpType), simpleOp("path", getPathOpType), simpleOp("set_?path", setPathOpType), simpleOp("del_?paths", delPathsOpType), simpleOp("to_?entries|toEntries", toEntriesOpType), simpleOp("from_?entries|fromEntries", fromEntriesOpType), simpleOp("with_?entries|withEntries", withEntriesOpType), simpleOp("with", withOpType), simpleOp("collect", collectOpType), simpleOp("del", deleteChildOpType), assignableOp("style", getStyleOpType, assignStyleOpType), assignableOp("tag|type", getTagOpType, assignTagOpType), simpleOp("kind", getKindOpType), assignableOp("anchor", getAnchorOpType, assignAnchorOpType), assignableOp("alias", getAliasOpType, assignAliasOpType), {"ALL_COMMENTS", `comments\s*=`, assignAllCommentsOp(false), 0}, {"ALL_COMMENTS_ASSIGN_RELATIVE", `comments\s*\|=`, assignAllCommentsOp(true), 0}, {"Block", `;`, opToken(blockOpType), 0}, {"Alternative", `\/\/`, opToken(alternativeOpType), 0}, {"DocumentIndex", `documentIndex|document_?index|di`, opToken(getDocumentIndexOpType), 0}, {"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0}, {"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0}, simpleOp("trim", trimOpType), simpleOp("to_?string", toStringOpType), {"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0}, {"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0}, {"FloatValue", `-?\d+(\.\d+)`, floatValue(), 0}, {"NumberValue", `-?\d+`, numberValue(), 0}, {"TrueBooleanValue", `[Tt][Rr][Uu][Ee]`, booleanValue(true), 0}, {"FalseBooleanValue", `[Ff][Aa][Ll][Ss][Ee]`, booleanValue(false), 0}, {"NullValue", `[Nn][Uu][Ll][Ll]|~`, nullValue(), 0}, {"QuotedStringValue", `"([^"\\]*(\\.[^"\\]*)*)"`, stringValue(), 0}, {"StrEnvOp", `strenv\([^\)]+\)`, envOp(true), 0}, {"EnvOp", `env\([^\)]+\)`, envOp(false), 0}, {"EnvSubstWithOptions", `envsubst\((ne|nu|ff| |,)+\)`, envSubstWithOptions(), 0}, simpleOp("envsubst", envsubstOpType), {"Equals", `\s*==\s*`, opToken(equalsOpType), 0}, {"NotEquals", `\s*!=\s*`, opToken(notEqualsOpType), 0}, {"GreaterThanEquals", `\s*>=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}), 0}, {"LessThanEquals", `\s*<=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}), 0}, {"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0}, {"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0}, simpleOp("min", minOpType), simpleOp("max", maxOpType), {"AssignRelative", `\|=[c]*`, assignOpToken(true), 0}, {"Assign", `=[c]*`, assignOpToken(false), 0}, {`whitespace`, `[ \t\n]+`, nil, 0}, {"WrappedPathElement", `\."[^ "]+"\??`, pathToken(true), 0}, {"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n!]+\??`, pathToken(false), 0}, {"Pipe", `\|`, opToken(pipeOpType), 0}, {"Self", `\.`, opToken(selfReferenceOpType), 0}, {"Union", `,`, opToken(unionOpType), 0}, {"MultiplyAssign", `\*=[\+|\?cdn]*`, multiplyWithPrefs(multiplyAssignOpType), 0}, {"Multiply", `\*[\+|\?cdn]*`, multiplyWithPrefs(multiplyOpType), 0}, {"Divide", `\/`, opToken(divideOpType), 0}, {"Modulo", `%`, opToken(moduloOpType), 0}, {"AddAssign", `\+=`, opToken(addAssignOpType), 0}, {"Add", `\+`, opToken(addOpType), 0}, {"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0}, {"Subtract", `\-`, opToken(subtractOpType), 0}, {"Comment", `#.*`, nil, 0}, simpleOp("pivot", pivotOpType), } type yqAction func(lexer.Token) (*token, error) type participleYqRule struct { Name string Pattern string CreateYqToken yqAction ParticipleTokenType lexer.TokenType } type participleLexer struct { lexerDefinition lexer.StringDefinition } func simpleOp(name string, opType *operationType) *participleYqRule { return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0} } func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule { return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0} } func newParticipleLexer() expressionTokeniser { simpleRules := make([]lexer.SimpleRule, len(participleYqRules)) for i, yqRule := range participleYqRules { simpleRules[i] = lexer.SimpleRule{Name: yqRule.Name, Pattern: yqRule.Pattern} } lexerDefinition := lexer.MustSimple(simpleRules) symbols := lexerDefinition.Symbols() for _, yqRule := range participleYqRules { yqRule.ParticipleTokenType = symbols[yqRule.Name] } return &participleLexer{lexerDefinition} } func pathToken(wrapped bool) yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value prefs := traversePreferences{} if value[len(value)-1:] == "?" { prefs.OptionalTraverse = true value = value[:len(value)-1] } value = value[1:] if wrapped { value = unwrap(value) } log.Debug("PathToken %v", value) op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil } } func recursiveDecentOpToken(includeMapKeys bool) yqAction { prefs := recursiveDescentPreferences{ RecurseArray: true, TraversePreferences: traversePreferences{ DontFollowAlias: true, IncludeMapKeys: includeMapKeys, }, } return opTokenWithPrefs(recursiveDescentOpType, nil, prefs) } func opTokenWithPrefs(opType *operationType, assignOpType *operationType, preferences interface{}) yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value op := &Operation{OperationType: opType, Value: opType.Type, StringValue: value, Preferences: preferences} var assign *Operation if assignOpType != nil { assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences} } return &token{TokenType: operationToken, Operation: op, AssignOperation: assign, CheckForPostTraverse: op.OperationType.CheckForPostTraverse}, nil } } func expressionOpToken(expression string) yqAction { return func(_ lexer.Token) (*token, error) { prefs := expressionOpPreferences{expression: expression} expressionOp := &Operation{OperationType: expressionOpType, Preferences: prefs} return &token{TokenType: operationToken, Operation: expressionOp}, nil } } func flattenWithDepth() yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value var depth, errParsingInt = extractNumberParameter(value) if errParsingInt != nil { return nil, errParsingInt } prefs := flattenPreferences{depth: depth} op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: flattenOpType.CheckForPostTraverse}, nil } } func assignAllCommentsOp(updateAssign bool) yqAction { return func(rawToken lexer.Token) (*token, error) { log.Debug("assignAllCommentsOp %v", rawToken.Value) value := rawToken.Value op := &Operation{ OperationType: assignCommentOpType, Value: assignCommentOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true}, } return &token{TokenType: operationToken, Operation: op}, nil } } func assignOpToken(updateAssign bool) yqAction { return func(rawToken lexer.Token) (*token, error) { log.Debug("assignOpToken %v", rawToken.Value) value := rawToken.Value prefs := assignPreferences{DontOverWriteAnchor: true} if strings.Contains(value, "c") { prefs.ClobberCustomTags = true } op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } } func booleanValue(val bool) yqAction { return func(rawToken lexer.Token) (*token, error) { return &token{TokenType: operationToken, Operation: createValueOperation(val, rawToken.Value)}, nil } } func nullValue() yqAction { return func(rawToken lexer.Token) (*token, error) { return &token{TokenType: operationToken, Operation: createValueOperation(nil, rawToken.Value)}, nil } } func stringValue() yqAction { return func(rawToken lexer.Token) (*token, error) { log.Debug("rawTokenvalue: %v", rawToken.Value) value := unwrap(rawToken.Value) log.Debug("unwrapped: %v", value) value = processEscapeCharacters(value) return &token{TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, StringValue: value, Value: value, }}, nil } } func envOp(strenv bool) yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value preferences := envOpPreferences{} if strenv { // strenv( ) value = value[7 : len(value)-1] preferences.StringValue = true } else { //env( ) value = value[4 : len(value)-1] } envOperation := createValueOperation(value, value) envOperation.OperationType = envOpType envOperation.Preferences = preferences return &token{TokenType: operationToken, Operation: envOperation, CheckForPostTraverse: envOpType.CheckForPostTraverse}, nil } } func envSubstWithOptions() yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value noEmpty := hasOptionParameter(value, "ne") noUnset := hasOptionParameter(value, "nu") failFast := hasOptionParameter(value, "ff") envsubstOpType.Type = "ENVSUBST" prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast} if noEmpty { envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY" } if noUnset { envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET" } op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } } func multiplyWithPrefs(op *operationType) yqAction { return func(rawToken lexer.Token) (*token, error) { prefs := multiplyPreferences{} prefs.AssignPrefs = assignPreferences{} options := rawToken.Value if strings.Contains(options, "+") { prefs.AppendArrays = true } if strings.Contains(options, "?") { prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} } if strings.Contains(options, "n") { prefs.AssignPrefs.OnlyWriteNull = true } if strings.Contains(options, "d") { prefs.DeepMergeArrays = true } if strings.Contains(options, "c") { prefs.AssignPrefs.ClobberCustomTags = true } prefs.TraversePrefs.DontFollowAlias = true prefs.TraversePrefs.ExactKeyMatch = true op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } } func getVariableOpToken() yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value value = value[1:] getVarOperation := createValueOperation(value, value) getVarOperation.OperationType = getVariableOpType return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil } } func hexValue() yqAction { return func(rawToken lexer.Token) (*token, error) { var originalString = rawToken.Value var numberString = originalString[2:] log.Debugf("numberString: %v", numberString) var number, errParsingInt = strconv.ParseInt(numberString, 16, 64) if errParsingInt != nil { return nil, errParsingInt } return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil } } func floatValue() yqAction { return func(rawToken lexer.Token) (*token, error) { var numberString = rawToken.Value var number, errParsingInt = strconv.ParseFloat(numberString, 64) if errParsingInt != nil { return nil, errParsingInt } return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil } } func numberValue() yqAction { return func(rawToken lexer.Token) (*token, error) { var numberString = rawToken.Value var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) if errParsingInt != nil { return nil, errParsingInt } return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil } } func parentWithLevel() yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value var level, errParsingInt = extractNumberParameter(value) if errParsingInt != nil { return nil, errParsingInt } prefs := parentOpPreferences{Level: level} op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil } } func parentWithDefaultLevel() yqAction { return func(_ lexer.Token) (*token, error) { prefs := parentOpPreferences{Level: 1} op := &Operation{OperationType: getParentOpType, Value: getParentOpType.Type, StringValue: getParentOpType.Type, Preferences: prefs} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil } } func encodeParseIndent(outputFormat *Format) yqAction { return func(rawToken lexer.Token) (*token, error) { value := rawToken.Value var indent, errParsingInt = extractNumberParameter(value) if errParsingInt != nil { return nil, errParsingInt } prefs := encoderPreferences{format: outputFormat, indent: indent} op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } } func encodeWithIndent(outputFormat *Format, indent int) yqAction { prefs := encoderPreferences{format: outputFormat, indent: indent} return opTokenWithPrefs(encodeOpType, nil, prefs) } func decodeOp(format *Format) yqAction { prefs := decoderPreferences{format: format} return opTokenWithPrefs(decodeOpType, nil, prefs) } func loadOp(decoder Decoder) yqAction { prefs := loadPrefs{decoder} return opTokenWithPrefs(loadOpType, nil, prefs) } func opToken(op *operationType) yqAction { return opTokenWithPrefs(op, nil, nil) } func literalToken(tt tokenType, checkForPost bool) yqAction { return func(rawToken lexer.Token) (*token, error) { return &token{TokenType: tt, CheckForPostTraverse: checkForPost, Match: rawToken.Value}, nil } } func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule { for _, yqRule := range participleYqRules { if yqRule.ParticipleTokenType == rawToken.Type { return yqRule } } return &participleYqRule{} } func (p *participleLexer) Tokenise(expression string) ([]*token, error) { myLexer, err := p.lexerDefinition.LexString("", expression) if err != nil { return nil, err } tokens := make([]*token, 0) for { rawToken, e := myLexer.Next() if e != nil { return nil, e } else if rawToken.Type == lexer.EOF { return postProcessTokens(tokens), nil } definition := p.getYqDefinition(rawToken) if definition.CreateYqToken != nil { token, e := definition.CreateYqToken(rawToken) if e != nil { return nil, e } tokens = append(tokens, token) } } } ================================================ FILE: pkg/yqlib/lexer_participle_test.go ================================================ package yqlib import ( "testing" "github.com/alecthomas/repr" "github.com/mikefarah/yq/v4/test" ) type participleLexerScenario struct { expression string tokens []*token } var participleLexerScenarios = []participleLexerScenario{ { expression: "to_entries[]", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: toEntriesOpType, Value: "TO_ENTRIES", StringValue: "to_entries", }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traverseArrayOpType, }, }, { TokenType: openCollect, Match: "[", }, { TokenType: operationToken, Operation: &Operation{ OperationType: emptyOpType, StringValue: "EMPTY", }, }, { TokenType: closeCollect, CheckForPostTraverse: true, Match: "]", }, }, }, { expression: ".a!=", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: notEqualsOpType, Value: "NOT_EQUALS", StringValue: "!=", }, }, }, }, { expression: ".[:3]", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: selfReferenceOpType, StringValue: "SELF", }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY", }, }, { TokenType: openCollect, }, { TokenType: operationToken, Operation: &Operation{ OperationType: valueOpType, Value: 0, StringValue: "0", CandidateNode: &CandidateNode{ Kind: ScalarNode, Tag: "!!int", Value: "0", }, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: createMapOpType, Value: "CREATE_MAP", StringValue: ":", }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: valueOpType, Value: int64(3), StringValue: "3", CandidateNode: &CandidateNode{ Kind: ScalarNode, Tag: "!!int", Value: "3", }, }, }, { TokenType: closeCollect, CheckForPostTraverse: true, Match: "]", }, }, }, { expression: ".[-2:]", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: selfReferenceOpType, StringValue: "SELF", }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY", }, }, { TokenType: openCollect, }, { TokenType: operationToken, Operation: &Operation{ OperationType: valueOpType, Value: int64(-2), StringValue: "-2", CandidateNode: &CandidateNode{ Kind: ScalarNode, Tag: "!!int", Value: "-2", }, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: createMapOpType, Value: "CREATE_MAP", StringValue: ":", }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: lengthOpType, }, }, { TokenType: closeCollect, CheckForPostTraverse: true, Match: "]", }, }, }, { expression: ".a", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: ".a.b", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: shortPipeOpType, Value: "PIPE", StringValue: ".", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b", StringValue: "b", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: ".a.b?", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: shortPipeOpType, Value: "PIPE", StringValue: ".", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b", StringValue: "b", Preferences: traversePreferences{ OptionalTraverse: true, }, }, CheckForPostTraverse: true, }, }, }, { expression: `.a."b?"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: shortPipeOpType, Value: "PIPE", StringValue: ".", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b?", StringValue: "b?", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: ` .a ."b?"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: shortPipeOpType, Value: "PIPE", StringValue: ".", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b?", StringValue: "b?", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: `.a | .b`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: pipeOpType, Value: "PIPE", StringValue: "|", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b", StringValue: "b", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: "(.a)", tokens: []*token{ { TokenType: openBracket, Match: "(", }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: closeBracket, Match: ")", CheckForPostTraverse: true, }, }, }, { expression: "..", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: recursiveDescentOpType, Value: "RECURSIVE_DESCENT", StringValue: "..", Preferences: recursiveDescentPreferences{ RecurseArray: true, TraversePreferences: traversePreferences{ DontFollowAlias: true, IncludeMapKeys: false, }, }, }, }, }, }, { expression: "...", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: recursiveDescentOpType, Value: "RECURSIVE_DESCENT", StringValue: "...", Preferences: recursiveDescentPreferences{ RecurseArray: true, TraversePreferences: traversePreferences{ DontFollowAlias: true, IncludeMapKeys: true, }, }, }, }, }, }, { expression: ".a,.b", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "a", StringValue: "a", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, { TokenType: operationToken, Operation: &Operation{ OperationType: unionOpType, Value: "UNION", StringValue: ",", Preferences: nil, }, }, { TokenType: operationToken, Operation: &Operation{ OperationType: traversePathOpType, Value: "b", StringValue: "b", Preferences: traversePreferences{}, }, CheckForPostTraverse: true, }, }, }, { expression: "map_values", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: mapValuesOpType, Value: "MAP_VALUES", StringValue: "map_values", Preferences: nil, }, CheckForPostTraverse: mapValuesOpType.CheckForPostTraverse, }, }, }, { expression: "mapvalues", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: mapValuesOpType, Value: "MAP_VALUES", StringValue: "mapvalues", Preferences: nil, }, CheckForPostTraverse: mapValuesOpType.CheckForPostTraverse, }, }, }, { expression: "flatten(3)", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: flattenOpType, Value: "FLATTEN_BY", StringValue: "flatten(3)", Preferences: flattenPreferences{depth: 3}, }, CheckForPostTraverse: flattenOpType.CheckForPostTraverse, }, }, }, { expression: "flatten", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: flattenOpType, Value: "FLATTEN_BY", StringValue: "flatten", Preferences: flattenPreferences{depth: -1}, }, CheckForPostTraverse: flattenOpType.CheckForPostTraverse, }, }, }, { expression: "length", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: lengthOpType, Value: "LENGTH", StringValue: "length", Preferences: nil, }, }, }, }, { expression: "format_datetime", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: formatDateTimeOpType, Value: "FORMAT_DATE_TIME", StringValue: "format_datetime", Preferences: nil, }, }, }, }, { expression: "to_yaml(3)", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: encodeOpType, Value: "ENCODE", StringValue: "to_yaml(3)", Preferences: encoderPreferences{ format: YamlFormat, indent: 3, }, }, }, }, }, { expression: "tojson(2)", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: encodeOpType, Value: "ENCODE", StringValue: "tojson(2)", Preferences: encoderPreferences{ format: JSONFormat, indent: 2, }, }, }, }, }, { expression: "@yaml", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: encodeOpType, Value: "ENCODE", StringValue: "@yaml", Preferences: encoderPreferences{ format: YamlFormat, indent: 2, }, }, }, }, }, { expression: "to_props", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: encodeOpType, Value: "ENCODE", StringValue: "to_props", Preferences: encoderPreferences{ format: PropertiesFormat, indent: 2, }, }, }, }, }, { expression: "@base64d", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: decodeOpType, Value: "DECODE", StringValue: "@base64d", Preferences: decoderPreferences{ format: Base64Format, }, }, }, }, }, { expression: "@base64", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: encodeOpType, Value: "ENCODE", StringValue: "@base64", Preferences: encoderPreferences{ format: Base64Format, }, }, }, }, }, { expression: "@yamld", tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: decodeOpType, Value: "DECODE", StringValue: "@yamld", Preferences: decoderPreferences{ format: YamlFormat, }, }, }, }, }, { expression: `"string with a\n"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\n", StringValue: "string with a\n", Preferences: nil, }, }, }, }, { expression: `"string with a \""`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: `string with a "`, StringValue: `string with a "`, Preferences: nil, }, }, }, }, { expression: `"string with a\r"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\r", StringValue: "string with a\r", Preferences: nil, }, }, }, }, { expression: `"string with a\t"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\t", StringValue: "string with a\t", Preferences: nil, }, }, }, }, { expression: `"string with a\f"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\f", StringValue: "string with a\f", Preferences: nil, }, }, }, }, { expression: `"string with a\v"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\v", StringValue: "string with a\v", Preferences: nil, }, }, }, }, { expression: `"string with a\b"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\b", StringValue: "string with a\b", Preferences: nil, }, }, }, }, { expression: `"string with a\a"`, tokens: []*token{ { TokenType: operationToken, Operation: &Operation{ OperationType: stringInterpolationOpType, Value: "string with a\a", StringValue: "string with a\a", Preferences: nil, }, }, }, }, } func TestParticipleLexer(t *testing.T) { lexer := newParticipleLexer() for _, scenario := range participleLexerScenarios { actual, err := lexer.Tokenise(scenario.expression) if err != nil { t.Error(err) } else { test.AssertResultWithContext(t, repr.String(scenario.tokens, repr.Indent(" ")), repr.String(actual, repr.Indent(" ")), scenario.expression) } } } ================================================ FILE: pkg/yqlib/lib.go ================================================ // Use the top level Evaluator or StreamEvaluator to evaluate expressions and return matches. package yqlib import ( "container/list" "fmt" "math" "strconv" "strings" logging "gopkg.in/op/go-logging.v1" ) var ExpressionParser ExpressionParserInterface func InitExpressionParser() { if ExpressionParser == nil { ExpressionParser = newExpressionParser() } } var log = logging.MustGetLogger("yq-lib") var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""` // GetLogger returns the yq logger instance. func GetLogger() *logging.Logger { return log } func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode { for index := 0; index < len(content); index = index + 2 { keyNode := content[index] valueNode := content[index+1] if keyNode.Value == key { return valueNode } } return nil } func recurseNodeArrayEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false } for index := 0; index < len(lhs.Content); index = index + 1 { if !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) { return false } } return true } func findInArray(array *CandidateNode, item *CandidateNode) int { for index := 0; index < len(array.Content); index = index + 1 { if recursiveNodeEqual(array.Content[index], item) { return index } } return -1 } func findKeyInMap(dataMap *CandidateNode, item *CandidateNode) int { for index := 0; index < len(dataMap.Content); index = index + 2 { if recursiveNodeEqual(dataMap.Content[index], item) { return index } } return -1 } func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if len(lhs.Content) != len(rhs.Content) { return false } for index := 0; index < len(lhs.Content); index = index + 2 { key := lhs.Content[index] value := lhs.Content[index+1] indexInRHS := findInArray(rhs, key) if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) { return false } } return true } func parseSnippet(value string) (*CandidateNode, error) { if value == "" { return &CandidateNode{ Kind: ScalarNode, Tag: "!!null", }, nil } decoder := NewYamlDecoder(ConfiguredYamlPreferences) err := decoder.Init(strings.NewReader(value)) if err != nil { return nil, err } result, err := decoder.Decode() if err != nil { return nil, err } if result.Kind == ScalarNode { result.LineComment = result.LeadingContent } else { result.HeadComment = result.LeadingContent } result.LeadingContent = "" if result.Tag == "!!str" { // use the original string value, as // decoding drops new lines newNode := createScalarNode(value, value) newNode.LineComment = result.LineComment return newNode, nil } result.Line = 0 result.Column = 0 return result, err } func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool { if lhs.Kind != rhs.Kind { return false } if lhs.Kind == ScalarNode { //process custom tags of scalar nodes. //dont worry about matching tags of maps or arrays. lhsTag := lhs.guessTagFromCustomType() rhsTag := rhs.guessTagFromCustomType() if lhsTag != rhsTag { return false } } if lhs.Tag == "!!null" { return true } else if lhs.Kind == ScalarNode { return lhs.Value == rhs.Value } else if lhs.Kind == SequenceNode { return recurseNodeArrayEqual(lhs, rhs) } else if lhs.Kind == MappingNode { return recurseNodeObjectEqual(lhs, rhs) } return false } // yaml numbers can have underscores, be hex and octal encoded... func parseInt64(numberString string) (string, int64, error) { if strings.Contains(numberString, "_") { numberString = strings.ReplaceAll(numberString, "_", "") } if strings.HasPrefix(numberString, "0x") || strings.HasPrefix(numberString, "0X") { num, err := strconv.ParseInt(numberString[2:], 16, 64) return "0x%X", num, err } else if strings.HasPrefix(numberString, "0o") { num, err := strconv.ParseInt(numberString[2:], 8, 64) return "0o%o", num, err } num, err := strconv.ParseInt(numberString, 10, 64) return "%v", num, err } func parseInt(numberString string) (int, error) { _, parsed, err := parseInt64(numberString) if err != nil { return 0, err } else if parsed > math.MaxInt || parsed < math.MinInt { return 0, fmt.Errorf("%v is not within [%v, %v]", parsed, math.MinInt, math.MaxInt) } return int(parsed), err } func processEscapeCharacters(original string) string { if original == "" { return original } var result strings.Builder runes := []rune(original) for i := 0; i < len(runes); i++ { if runes[i] == '\\' && i < len(runes)-1 { next := runes[i+1] switch next { case '\\': // Check if followed by opening bracket - if so, preserve both backslashes // this is required for string interpolation to work correctly. if i+2 < len(runes) && runes[i+2] == '(' { // Preserve \\ when followed by ( result.WriteRune('\\') result.WriteRune('\\') i++ // Skip the next backslash (we'll process the ( normally on next iteration) continue } // Escaped backslash: \\ -> \ result.WriteRune('\\') i++ // Skip the next backslash continue case '"': result.WriteRune('"') i++ // Skip the quote continue case 'n': result.WriteRune('\n') i++ // Skip the 'n' continue case 't': result.WriteRune('\t') i++ // Skip the 't' continue case 'r': result.WriteRune('\r') i++ // Skip the 'r' continue case 'f': result.WriteRune('\f') i++ // Skip the 'f' continue case 'v': result.WriteRune('\v') i++ // Skip the 'v' continue case 'b': result.WriteRune('\b') i++ // Skip the 'b' continue case 'a': result.WriteRune('\a') i++ // Skip the 'a' continue } } result.WriteRune(runes[i]) } value := result.String() if value != original { log.Debug("processEscapeCharacters from [%v] to [%v]", original, value) } return value } func headAndLineComment(node *CandidateNode) string { return headComment(node) + lineComment(node) } func headComment(node *CandidateNode) string { return strings.Replace(node.HeadComment, "#", "", 1) } func lineComment(node *CandidateNode) string { return strings.Replace(node.LineComment, "#", "", 1) } func footComment(node *CandidateNode) string { return strings.Replace(node.FootComment, "#", "", 1) } // use for debugging only func NodesToString(collection *list.List) string { if !log.IsEnabledFor(logging.DEBUG) { return "" } result := fmt.Sprintf("%v results\n", collection.Len()) for el := collection.Front(); el != nil; el = el.Next() { result = result + "\n" + NodeToString(el.Value.(*CandidateNode)) } return result } func NodeToString(node *CandidateNode) string { if !log.IsEnabledFor(logging.DEBUG) { return "" } if node == nil { return "-- nil --" } tag := node.Tag if node.Kind == AliasNode { tag = "alias" } valueToUse := node.Value if valueToUse == "" { valueToUse = fmt.Sprintf("%v kids", len(node.Content)) } return fmt.Sprintf(`D%v, P%v, %v (%v)::%v`, node.GetDocument(), node.GetNicePath(), KindString(node.Kind), tag, valueToUse) } func NodeContentToString(node *CandidateNode, depth int) string { if !log.IsEnabledFor(logging.DEBUG) { return "" } var sb strings.Builder for _, child := range node.Content { for i := 0; i < depth; i++ { sb.WriteString(" ") } sb.WriteString("- ") sb.WriteString(NodeToString(child)) sb.WriteString("\n") sb.WriteString(NodeContentToString(child, depth+1)) } return sb.String() } func KindString(kind Kind) string { switch kind { case ScalarNode: return "ScalarNode" case SequenceNode: return "SequenceNode" case MappingNode: return "MappingNode" case AliasNode: return "AliasNode" default: return "unknown!" } } ================================================ FILE: pkg/yqlib/lib_test.go ================================================ package yqlib import ( "fmt" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) func TestGetLogger(t *testing.T) { l := GetLogger() if l != log { t.Fatal("GetLogger should return the yq logger instance, not a copy") } } type parseSnippetScenario struct { snippet string expected *CandidateNode expectedError string } var parseSnippetScenarios = []parseSnippetScenario{ { snippet: ":", expectedError: "yaml: did not find expected key", }, { snippet: "", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!null", }, }, { snippet: "null", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!null", Value: "null", Line: 0, Column: 0, }, }, { snippet: "3", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!int", Value: "3", Line: 0, Column: 0, }, }, { snippet: "cat", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "cat", Line: 0, Column: 0, }, }, { snippet: "# things", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!null", LineComment: "# things", Line: 0, Column: 0, }, }, { snippet: "3.1", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!float", Value: "3.1", Line: 0, Column: 0, }, }, { snippet: "true", expected: &CandidateNode{ Kind: ScalarNode, Tag: "!!bool", Value: "true", Line: 0, Column: 0, }, }, } func TestParseSnippet(t *testing.T) { for _, tt := range parseSnippetScenarios { actual, err := parseSnippet(tt.snippet) if tt.expectedError != "" { if err == nil { t.Errorf("Expected error '%v' but it worked!", tt.expectedError) } else { test.AssertResultComplexWithContext(t, tt.expectedError, err.Error(), tt.snippet) } continue } if err != nil { t.Error(tt.snippet) t.Error(err) } test.AssertResultComplexWithContext(t, tt.expected, actual, tt.snippet) } } type parseInt64Scenario struct { numberString string expectedParsedNumber int64 expectedFormatString string } var parseInt64Scenarios = []parseInt64Scenario{ { numberString: "34", expectedParsedNumber: 34, }, { numberString: "10_000", expectedParsedNumber: 10000, expectedFormatString: "10000", }, { numberString: "0x10", expectedParsedNumber: 16, }, { numberString: "0x10_000", expectedParsedNumber: 65536, expectedFormatString: "0x10000", }, { numberString: "0o10", expectedParsedNumber: 8, }, } func TestParseInt64(t *testing.T) { for _, tt := range parseInt64Scenarios { format, actualNumber, err := parseInt64(tt.numberString) if err != nil { t.Error(tt.numberString) t.Error(err) } test.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString) if tt.expectedFormatString == "" { tt.expectedFormatString = tt.numberString } test.AssertResultComplexWithContext(t, tt.expectedFormatString, fmt.Sprintf(format, actualNumber), fmt.Sprintf("Formatting of: %v", tt.numberString)) } } func TestGetContentValueByKey(t *testing.T) { // Create content with key-value pairs key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") content := []*CandidateNode{key1, value1, key2, value2} // Test finding existing key result := getContentValueByKey(content, "key1") test.AssertResult(t, value1, result) // Test finding another existing key result = getContentValueByKey(content, "key2") test.AssertResult(t, value2, result) // Test finding non-existing key result = getContentValueByKey(content, "nonexistent") test.AssertResult(t, (*CandidateNode)(nil), result) // Test with empty content result = getContentValueByKey([]*CandidateNode{}, "key1") test.AssertResult(t, (*CandidateNode)(nil), result) } func TestRecurseNodeArrayEqual(t *testing.T) { // Create two arrays with same content array1 := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{ createStringScalarNode("item1"), createStringScalarNode("item2"), }, } array2 := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{ createStringScalarNode("item1"), createStringScalarNode("item2"), }, } array3 := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{ createStringScalarNode("item1"), createStringScalarNode("different"), }, } array4 := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{ createStringScalarNode("item1"), }, } test.AssertResult(t, true, recurseNodeArrayEqual(array1, array2)) test.AssertResult(t, false, recurseNodeArrayEqual(array1, array3)) test.AssertResult(t, false, recurseNodeArrayEqual(array1, array4)) } func TestFindInArray(t *testing.T) { item1 := createStringScalarNode("item1") item2 := createStringScalarNode("item2") item3 := createStringScalarNode("item3") array := &CandidateNode{ Kind: SequenceNode, Content: []*CandidateNode{item1, item2, item3}, } // Test finding existing items test.AssertResult(t, 0, findInArray(array, item1)) test.AssertResult(t, 1, findInArray(array, item2)) test.AssertResult(t, 2, findInArray(array, item3)) // Test finding non-existing item nonExistent := createStringScalarNode("nonexistent") test.AssertResult(t, -1, findInArray(array, nonExistent)) } func TestFindKeyInMap(t *testing.T) { key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") mapNode := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key2, value2}, } // Test finding existing keys test.AssertResult(t, 0, findKeyInMap(mapNode, key1)) test.AssertResult(t, 2, findKeyInMap(mapNode, key2)) // Test finding non-existing key nonExistent := createStringScalarNode("nonexistent") test.AssertResult(t, -1, findKeyInMap(mapNode, nonExistent)) } func TestRecurseNodeObjectEqual(t *testing.T) { // Create two objects with same content key1 := createStringScalarNode("key1") value1 := createStringScalarNode("value1") key2 := createStringScalarNode("key2") value2 := createStringScalarNode("value2") obj1 := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key2, value2}, } obj2 := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key2, value2}, } // Create object with different values value3 := createStringScalarNode("value3") obj3 := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value3, key2, value2}, } // Create object with different keys key3 := createStringScalarNode("key3") obj4 := &CandidateNode{ Kind: MappingNode, Content: []*CandidateNode{key1, value1, key3, value2}, } test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2)) test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3)) test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4)) } func TestParseInt(t *testing.T) { type parseIntScenario struct { numberString string expectedParsedNumber int expectedError string } scenarios := []parseIntScenario{ { numberString: "34", expectedParsedNumber: 34, }, { numberString: "10_000", expectedParsedNumber: 10000, }, { numberString: "0x10", expectedParsedNumber: 16, }, { numberString: "0o10", expectedParsedNumber: 8, }, { numberString: "invalid", expectedError: "strconv.ParseInt", }, } for _, tt := range scenarios { actualNumber, err := parseInt(tt.numberString) if tt.expectedError != "" { if err == nil { t.Errorf("Expected error for '%s' but got none", tt.numberString) } else if !strings.Contains(err.Error(), tt.expectedError) { t.Errorf("Expected error containing '%s' for '%s', got '%s'", tt.expectedError, tt.numberString, err.Error()) } continue } if err != nil { t.Errorf("Unexpected error for '%s': %v", tt.numberString, err) } test.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString) } } func TestHeadAndLineComment(t *testing.T) { node := &CandidateNode{ HeadComment: "# head comment", LineComment: "# line comment", } result := headAndLineComment(node) test.AssertResult(t, " head comment line comment", result) } func TestHeadComment(t *testing.T) { node := &CandidateNode{ HeadComment: "# head comment", } result := headComment(node) test.AssertResult(t, " head comment", result) // Test without # node.HeadComment = "no hash comment" result = headComment(node) test.AssertResult(t, "no hash comment", result) } func TestLineComment(t *testing.T) { node := &CandidateNode{ LineComment: "# line comment", } result := lineComment(node) test.AssertResult(t, " line comment", result) // Test without # node.LineComment = "no hash comment" result = lineComment(node) test.AssertResult(t, "no hash comment", result) } func TestFootComment(t *testing.T) { node := &CandidateNode{ FootComment: "# foot comment", } result := footComment(node) test.AssertResult(t, " foot comment", result) // Test without # node.FootComment = "no hash comment" result = footComment(node) test.AssertResult(t, "no hash comment", result) } func TestKindString(t *testing.T) { test.AssertResult(t, "ScalarNode", KindString(ScalarNode)) test.AssertResult(t, "SequenceNode", KindString(SequenceNode)) test.AssertResult(t, "MappingNode", KindString(MappingNode)) test.AssertResult(t, "AliasNode", KindString(AliasNode)) test.AssertResult(t, "unknown!", KindString(Kind(999))) // Invalid kind } type processEscapeCharactersScenario struct { input string expected string } var processEscapeCharactersScenarios = []processEscapeCharactersScenario{ { input: "", expected: "", }, { input: "hello", expected: "hello", }, { input: "\\\"", expected: "\"", }, { input: "hello\\\"world", expected: "hello\"world", }, { input: "\\n", expected: "\n", }, { input: "line1\\nline2", expected: "line1\nline2", }, { input: "\\t", expected: "\t", }, { input: "hello\\tworld", expected: "hello\tworld", }, { input: "\\r", expected: "\r", }, { input: "hello\\rworld", expected: "hello\rworld", }, { input: "\\f", expected: "\f", }, { input: "hello\\fworld", expected: "hello\fworld", }, { input: "\\v", expected: "\v", }, { input: "hello\\vworld", expected: "hello\vworld", }, { input: "\\b", expected: "\b", }, { input: "hello\\bworld", expected: "hello\bworld", }, { input: "\\a", expected: "\a", }, { input: "hello\\aworld", expected: "hello\aworld", }, { input: "\\\"\\n\\t\\r\\f\\v\\b\\a", expected: "\"\n\t\r\f\v\b\a", }, { input: "multiple\\nlines\\twith\\ttabs", expected: "multiple\nlines\twith\ttabs", }, { input: "quote\\\"here", expected: "quote\"here", }, { input: "\\\\", expected: "\\", // Backslash is processed: "\\\\" becomes "\\" }, { input: "\\\"test\\\"", expected: "\"test\"", }, { input: "a\\\\b", expected: "a\\b", // Tests roundtrip: "a\\\\b" should become "a\\b" }, { input: "Hi \\\\(.value)", expected: "Hi \\\\(.value)", }, { input: `a\\b`, expected: "a\\b", }, } func TestProcessEscapeCharacters(t *testing.T) { for _, tt := range processEscapeCharactersScenarios { actual := processEscapeCharacters(tt.input) test.AssertResultComplexWithContext(t, tt.expected, actual, fmt.Sprintf("Input: %q", tt.input)) } } ================================================ FILE: pkg/yqlib/lua.go ================================================ package yqlib type LuaPreferences struct { DocPrefix string DocSuffix string UnquotedKeys bool Globals bool } func NewDefaultLuaPreferences() LuaPreferences { return LuaPreferences{ DocPrefix: "return ", DocSuffix: ";\n", UnquotedKeys: false, Globals: false, } } var ConfiguredLuaPreferences = NewDefaultLuaPreferences() ================================================ FILE: pkg/yqlib/lua_test.go ================================================ //go:build !yq_nolua package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) var luaScenarios = []formatScenario{ { description: "Basic input example", input: `return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, expected: `country: Australia cities: - Sydney - Melbourne - Brisbane - Perth `, }, { skipDoc: true, description: "path", expression: ".cities[2] | path", input: `return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, expected: "- cities\n- 2\n", }, { skipDoc: true, description: "path", expression: ".cities[2] | key", input: `return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, expected: "2\n", }, { description: "Basic output example", scenarioType: "encode", input: `--- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth`, expected: `return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, }, { description: "Basic roundtrip", skipDoc: true, scenarioType: "roundtrip", expression: `.cities[0] = "Adelaide"`, input: `return { ["country"] = "Australia"; -- this place ["cities"] = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, expected: `return { ["country"] = "Australia"; ["cities"] = { "Adelaide", "Melbourne", "Brisbane", "Perth", }; }; `, }, { description: "Unquoted keys", subdescription: "Uses the `--lua-unquoted` option to produce a nicer-looking output.", scenarioType: "unquoted-encode", input: `--- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth`, expected: `return { country = "Australia"; -- this place cities = { "Sydney", "Melbourne", "Brisbane", "Perth", }; }; `, }, { description: "Globals", subdescription: "Uses the `--lua-globals` option to export the values into the global scope.", scenarioType: "globals-encode", input: `--- country: Australia # this place cities: - Sydney - Melbourne - Brisbane - Perth`, expected: `country = "Australia"; -- this place cities = { "Sydney", "Melbourne", "Brisbane", "Perth", }; `, }, { description: "Elaborate example", input: `--- hello: world tables: like: this keys: values ? look: non-string keys : True numbers: - decimal: 12345 - hex: 0x7fabc123 - octal: 0o30 - float: 123.45 - infinity: .inf plus_infinity: +.inf minus_infinity: -.inf - not: .nan `, expected: `return { ["hello"] = "world"; ["tables"] = { ["like"] = "this"; ["keys"] = "values"; [{ ["look"] = "non-string keys"; }] = true; }; ["numbers"] = { { ["decimal"] = 12345; }, { ["hex"] = 0x7fabc123; }, { ["octal"] = 24; }, { ["float"] = 123.45; }, { ["infinity"] = (1/0); ["plus_infinity"] = (1/0); ["minus_infinity"] = (-1/0); }, { ["not"] = (0/0); }, }; }; `, scenarioType: "encode", }, { skipDoc: true, description: "Sequence", input: "- a\n- b\n- c\n", expected: "return {\n\t\"a\",\n\t\"b\",\n\t\"c\",\n};\n", scenarioType: "encode", }, { skipDoc: true, description: "Mapping", input: "a: b\nc:\n d: e\nf: 0\n", expected: "return {\n\t[\"a\"] = \"b\";\n\t[\"c\"] = {\n\t\t[\"d\"] = \"e\";\n\t};\n\t[\"f\"] = 0;\n};\n", scenarioType: "encode", }, { skipDoc: true, description: "Scalar str", input: "str: |\n foo\n bar\nanother: 'single'\nand: \"double\"", expected: "return {\n\t[\"str\"] = [[\nfoo\nbar\n]];\n\t[\"another\"] = 'single';\n\t[\"and\"] = \"double\";\n};\n", scenarioType: "encode", }, { skipDoc: true, description: "Scalar null", input: "x: null\n", expected: "return {\n\t[\"x\"] = nil;\n};\n", scenarioType: "encode", }, { skipDoc: true, description: "Scalar int", input: "- 1\n- 2\n- 0x10\n- 0o30\n- -999\n", expected: "return {\n\t1,\n\t2,\n\t0x10,\n\t24,\n\t-999,\n};\n", scenarioType: "encode", }, { skipDoc: true, description: "Scalar float", input: "- 1.0\n- 3.14\n- 1e100\n- .Inf\n- .NAN\n", expected: "return {\n\t1.0,\n\t3.14,\n\t1e100,\n\t(1/0),\n\t(0/0),\n};\n", scenarioType: "encode", }, } func testLuaScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) case "unquoted-encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ DocPrefix: "return ", DocSuffix: ";\n", UnquotedKeys: true, Globals: false, })), s.description) case "globals-encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ DocPrefix: "return ", DocSuffix: ";\n", UnquotedKeys: false, Globals: true, })), s.description) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "decode": documentLuaDecodeScenario(w, s) case "encode", "unquoted-encode", "globals-encode": documentLuaEncodeScenario(w, s) case "roundtrip": documentLuaRoundTripScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.lua file of:\n") writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } prefs := ConfiguredLuaPreferences switch s.scenarioType { case "unquoted-encode": prefs = LuaPreferences{ DocPrefix: "return ", DocSuffix: ";\n", UnquotedKeys: true, Globals: false, } case "globals-encode": prefs = LuaPreferences{ DocPrefix: "return ", DocSuffix: ";\n", UnquotedKeys: false, Globals: true, } } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") switch s.scenarioType { case "unquoted-encode": writeOrPanic(w, "```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\n") case "globals-encode": writeOrPanic(w, "```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\n") default: writeOrPanic(w, "```bash\nyq -o=lua '.' sample.yml\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) } func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.lua file of:\n") writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq '.' sample.lua\n```\n") writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)))) } func TestLuaScenarios(t *testing.T) { for _, tt := range luaScenarios { testLuaScenario(t, tt) } genericScenarios := make([]interface{}, len(luaScenarios)) for i, s := range luaScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "lua", genericScenarios, documentLuaScenario) } ================================================ FILE: pkg/yqlib/matchKeyString.go ================================================ package yqlib func matchKey(name string, pattern string) (matched bool) { if pattern == "" { return name == pattern } if pattern == "*" { log.Debug("wild!") return true } return deepMatch(name, pattern) } // deepMatch reports whether the name matches the pattern in linear time. // Source https://research.swtch.com/glob func deepMatch(name, pattern string) bool { px := 0 nx := 0 nextPx := 0 nextNx := 0 for px < len(pattern) || nx < len(name) { if px < len(pattern) { c := pattern[px] switch c { default: // ordinary character if nx < len(name) && name[nx] == c { px++ nx++ continue } case '?': // single-character wildcard if nx < len(name) { px++ nx++ continue } case '*': // zero-or-more-character wildcard // Try to match at nx. // If that doesn't work out, // restart at nx+1 next. nextPx = px nextNx = nx + 1 px++ continue } } // Mismatch. Maybe restart. if 0 < nextNx && nextNx <= len(name) { px = nextPx nx = nextNx continue } return false } // Matched all of pattern to all of name. Success. return true } ================================================ FILE: pkg/yqlib/matchKeyString_test.go ================================================ package yqlib import ( "strings" "testing" ) func TestDeepMatch(t *testing.T) { var tests = []struct { name string pattern string ok bool }{ {"", "", true}, {"<<", "<<", true}, {"", "x", false}, {"x", "", false}, {"abc", "abc", true}, {"abc", "*", true}, {"abc", "*c", true}, {"abc", "*b", false}, {"abc", "a*", true}, {"abc", "b*", false}, {"a", "a*", true}, {"a", "*a", true}, {"axbxcxdxe", "a*b*c*d*e*", true}, {"axbxcxdxexxx", "a*b*c*d*e*", true}, {"abxbbxdbxebxczzx", "a*b?c*x", true}, {"abxbbxdbxebxczzy", "a*b?c*x", false}, {strings.Repeat("a", 100), "a*a*a*a*b", false}, {"xxx", "*x", true}, } for _, tt := range tests { t.Run(tt.name+" "+tt.pattern, func(t *testing.T) { if want, got := tt.ok, deepMatch(tt.name, tt.pattern); want != got { t.Errorf("Expected %v got %v", want, got) } }) } } ================================================ FILE: pkg/yqlib/no_base64.go ================================================ //go:build yq_nobase64 package yqlib func NewBase64Decoder() Decoder { return nil } func NewBase64Encoder() Encoder { return nil } ================================================ FILE: pkg/yqlib/no_csv.go ================================================ //go:build yq_nocsv package yqlib func NewCSVObjectDecoder(prefs CsvPreferences) Decoder { return nil } func NewCsvEncoder(prefs CsvPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_hcl.go ================================================ //go:build yq_nohcl package yqlib func NewHclDecoder() Decoder { return nil } func NewHclEncoder(_ HclPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_ini.go ================================================ //go:build yq_noini package yqlib func NewINIDecoder() Decoder { return nil } func NewINIEncoder() Encoder { return nil } ================================================ FILE: pkg/yqlib/no_json.go ================================================ //go:build yq_nojson package yqlib func NewJSONDecoder() Decoder { return nil } func NewJSONEncoder(prefs JsonPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_kyaml.go ================================================ //go:build yq_nokyaml package yqlib func NewKYamlEncoder(_ KYamlPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_lua.go ================================================ //go:build yq_nolua package yqlib func NewLuaEncoder(prefs LuaPreferences) Encoder { return nil } func NewLuaDecoder(prefs LuaPreferences) Decoder { return nil } ================================================ FILE: pkg/yqlib/no_props.go ================================================ //go:build yq_noprops package yqlib func NewPropertiesDecoder() Decoder { return nil } func NewPropertiesEncoder(prefs PropertiesPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_sh.go ================================================ //go:build yq_nosh package yqlib func NewShEncoder() Encoder { return nil } ================================================ FILE: pkg/yqlib/no_shellvariables.go ================================================ //go:build yq_noshell package yqlib func NewShellVariablesEncoder() Encoder { return nil } ================================================ FILE: pkg/yqlib/no_toml.go ================================================ //go:build yq_notoml package yqlib func NewTomlDecoder() Decoder { return nil } func NewTomlEncoder() Encoder { return nil } func NewTomlEncoderWithPrefs(prefs TomlPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/no_uri.go ================================================ //go:build yq_nouri package yqlib func NewUriDecoder() Decoder { return nil } func NewUriEncoder() Encoder { return nil } ================================================ FILE: pkg/yqlib/no_xml.go ================================================ //go:build yq_noxml package yqlib func NewXMLDecoder(prefs XmlPreferences) Decoder { return nil } func NewXMLEncoder(prefs XmlPreferences) Encoder { return nil } ================================================ FILE: pkg/yqlib/operation.go ================================================ package yqlib import "fmt" type Operation struct { OperationType *operationType Value interface{} StringValue string CandidateNode *CandidateNode // used for Value Path elements Preferences interface{} UpdateAssign bool // used for assign ops, when true it means we evaluate the rhs given the lhs } type operationType struct { Type string NumArgs uint // number of arguments to the op Precedence uint Handler operatorHandler CheckForPostTraverse bool ToString func(o *Operation) string } var valueToStringFunc = func(p *Operation) string { return fmt.Sprintf("%v (%T)", p.Value, p.Value) } func createValueOperation(value interface{}, stringValue string) *Operation { log.Debug("creating value op for string %v", stringValue) var node = createScalarNode(value, stringValue) return &Operation{ OperationType: valueOpType, Value: value, StringValue: stringValue, CandidateNode: node, } } var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator} var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator} var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator} var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator} var unionOpType = &operationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: unionOperator} var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handler: pipeOperator} var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator} var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator} var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator} var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator} var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator} var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe} var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator} var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator} var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator} var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precedence: 40, Handler: assignAliasOperator} var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator} var multiplyAssignOpType = &operationType{Type: "MULTIPLY_ASSIGN", NumArgs: 2, Precedence: 42, Handler: multiplyAssignOperator} var divideOpType = &operationType{Type: "DIVIDE", NumArgs: 2, Precedence: 42, Handler: divideOperator} var moduloOpType = &operationType{Type: "MODULO", NumArgs: 2, Precedence: 42, Handler: moduloOperator} var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator} var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator} var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator} var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator} var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator} var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator} var minOpType = &operationType{Type: "MIN", NumArgs: 0, Precedence: 40, Handler: minOperator} var maxOpType = &operationType{Type: "MAX", NumArgs: 0, Precedence: 40, Handler: maxOperator} // createmap needs to be above union, as we use union to build the components of the objects var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator} var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator} var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator} var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator} var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator} // Use this expression to create alias/syntactic sugar expressions (in lexer_participle). var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator} var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator} var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 52, Handler: mapOperator, CheckForPostTraverse: true} var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 52, Handler: filterOperator, CheckForPostTraverse: true} var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator} var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 52, Handler: pickOperator, CheckForPostTraverse: true} var omitOpType = &operationType{Type: "OMIT", NumArgs: 1, Precedence: 52, Handler: omitOperator, CheckForPostTraverse: true} var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 52, Handler: evalOperator, CheckForPostTraverse: true} var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 52, Handler: mapValuesOperator, CheckForPostTraverse: true} var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime} var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat} var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp} var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp} var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp} var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp} var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator} var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator} var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator} var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator} var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator} var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator} var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator} var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 52, Handler: toEntriesOperator, CheckForPostTraverse: true} var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator} var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator} var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 52, Handler: withOperator, CheckForPostTraverse: true} var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 52, Handler: splitDocumentOperator, CheckForPostTraverse: true} var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator} var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator} var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator} var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator} var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator} var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator} var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator} var getParentsOpType = &operationType{Type: "GET_PARENTS", NumArgs: 0, Precedence: 50, Handler: getParentsOperator} var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator} var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator} var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator} var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator} var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator} var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator} var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 52, Handler: getPathOperator, CheckForPostTraverse: true} var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator} var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 52, Handler: delPathsOperator, CheckForPostTraverse: true} var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 52, Handler: explodeOperator, CheckForPostTraverse: true} var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 52, Handler: sortByOperator, CheckForPostTraverse: true} var firstOpType = &operationType{Type: "FIRST", NumArgs: 1, Precedence: 52, Handler: firstOperator, CheckForPostTraverse: true} var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 52, Handler: reverseOperator, CheckForPostTraverse: true} var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 52, Handler: sortOperator, CheckForPostTraverse: true} var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 52, Handler: shuffleOperator, CheckForPostTraverse: true} var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 52, Handler: sortKeysOperator, CheckForPostTraverse: true} var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator} var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator} var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator} var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator} var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator} var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 52, Handler: splitStringOperator, CheckForPostTraverse: true} var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator} var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator} var toStringOpType = &operationType{Type: "TO_STRING", NumArgs: 0, Precedence: 50, Handler: toStringOperator} var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, Precedence: 50, Handler: stringInterpolationOperator, ToString: valueToStringFunc} var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 55, Handler: traversePathOperator, ToString: func(p *Operation) string { return fmt.Sprintf("%v", p.Value) }} var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Precedence: 50, Handler: traverseArrayOperator} var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator} var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator, ToString: valueToStringFunc} var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator} var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 52, Handler: envOperator, CheckForPostTraverse: true} var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator} var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator} var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator} var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator} var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator} var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 52, Handler: selectOperator, CheckForPostTraverse: true} var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator} var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 52, Handler: unique, CheckForPostTraverse: true} var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 52, Handler: uniqueBy, CheckForPostTraverse: true} var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 52, Handler: groupBy, CheckForPostTraverse: true} var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 52, Handler: flattenOp, CheckForPostTraverse: true} var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator} var pivotOpType = &operationType{Type: "PIVOT", NumArgs: 0, Precedence: 52, Handler: pivotOperator, CheckForPostTraverse: true} // debugging purposes only func (p *Operation) toString() string { if p == nil { return "OP IS NIL" } if p.OperationType.ToString != nil { return p.OperationType.ToString(p) } return fmt.Sprintf("%v", p.OperationType.Type) } ================================================ FILE: pkg/yqlib/operator_add.go ================================================ package yqlib import ( "fmt" "strconv" "strings" "time" ) func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { return &ExpressionNode{Operation: &Operation{OperationType: addOpType}, LHS: lhs, RHS: rhs} } func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { return compoundAssignFunction(d, context, expressionNode, createAddOp) } func toNodes(candidate *CandidateNode, lhs *CandidateNode) []*CandidateNode { if candidate.Tag == "!!null" { return []*CandidateNode{} } clone := candidate.Copy() switch candidate.Kind { case SequenceNode: return clone.Content default: if len(lhs.Content) > 0 { clone.Style = lhs.Content[0].Style } return []*CandidateNode{clone} } } func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Add operator") // only calculate when empty IF we are the root expression; OR // calcWhenEmpty := expressionNode.Parent == nil || expressionNode.Parent.LHS == expressionNode calcWhenEmpty := context.MatchingNodes.Len() > 0 return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, calcWhenEmpty) } func add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { lhsNode := lhs if lhs == nil && rhs == nil { return nil, nil } else if lhs == nil { return rhs.Copy(), nil } else if rhs == nil { return lhs.Copy(), nil } else if lhsNode.Tag == "!!null" { return lhs.CopyAsReplacement(rhs), nil } target := lhs.CopyWithoutContent() switch lhsNode.Kind { case MappingNode: if rhs.Kind != MappingNode { return nil, fmt.Errorf("%v (%v) cannot be added to a %v (%v)", rhs.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath()) } addMaps(target, lhs, rhs) case SequenceNode: addSequences(target, lhs, rhs) case ScalarNode: if rhs.Kind != ScalarNode { return nil, fmt.Errorf("%v (%v) cannot be added to a %v (%v)", rhs.Tag, rhs.GetNicePath(), lhsNode.Tag, lhs.GetNicePath()) } target.Kind = ScalarNode target.Style = lhsNode.Style if err := addScalars(context, target, lhsNode, rhs); err != nil { return nil, err } } return target, nil } func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { lhsTag := lhs.Tag rhsTag := rhs.guessTagFromCustomType() lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() lhsIsCustom = true } isDateTime := lhs.Tag == "!!timestamp" // if the lhs is a string, it might be a timestamp in a custom format. if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) isDateTime = err == nil } if isDateTime { return addDateTimes(context.GetDateTimeLayout(), target, lhs, rhs) } else if lhsTag == "!!str" { target.Tag = lhs.Tag if rhsTag == "!!null" { target.Value = lhs.Value } else { target.Value = lhs.Value + rhs.Value } } else if rhsTag == "!!str" { target.Tag = rhs.Tag target.Value = lhs.Value + rhs.Value } else if lhsTag == "!!int" && rhsTag == "!!int" { format, lhsNum, err := parseInt64(lhs.Value) if err != nil { return err } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { return err } sum := lhsNum + rhsNum target.Tag = lhs.Tag target.Value = fmt.Sprintf(format, sum) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return err } sum := lhsNum + rhsNum if lhsIsCustom { target.Tag = lhs.Tag } else { target.Tag = "!!float" } target.Value = fmt.Sprintf("%v", sum) } else { return fmt.Errorf("%v cannot be added to %v", lhsTag, rhsTag) } return nil } func addDateTimes(layout string, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { duration, err := time.ParseDuration(rhs.Value) if err != nil { return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) } currentTime, err := parseDateTime(layout, lhs.Value) if err != nil { return err } newTime := currentTime.Add(duration) target.Value = newTime.Format(layout) return nil } func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) { log.Debugf("adding sequences! target: %v; lhs %v; rhs: %v", NodeToString(target), NodeToString(lhs), NodeToString(rhs)) target.Kind = SequenceNode if len(lhs.Content) == 0 { log.Debugf("dont copy lhs style") target.Style = 0 } target.Tag = lhs.Tag extraNodes := toNodes(rhs, lhs) target.AddChildren(lhs.Content) target.AddChildren(extraNodes) } func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { lhs := lhsC rhs := rhsC if len(lhs.Content) == 0 { log.Debugf("dont copy lhs style") target.Style = 0 } target.Content = make([]*CandidateNode, 0) target.AddChildren(lhs.Content) for index := 0; index < len(rhs.Content); index = index + 2 { key := rhs.Content[index] value := rhs.Content[index+1] log.Debug("finding %v", key.Value) indexInLHS := findKeyInMap(target, key) log.Debug("indexInLhs %v", indexInLHS) if indexInLHS < 0 { // not in there, append it target.AddKeyValueChild(key, value) } else { // it's there, replace it oldValue := target.Content[indexInLHS+1] newValueCopy := oldValue.CopyAsReplacement(value) target.Content[indexInLHS+1] = newValueCopy } } target.Kind = MappingNode if len(lhs.Content) > 0 { target.Style = lhs.Style } target.Tag = lhs.Tag } ================================================ FILE: pkg/yqlib/operator_add_test.go ================================================ package yqlib import ( "testing" ) var addOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: `"foo" + "bar"`, expected: []string{ "D0, P[], (!!str)::foobar\n", }, }, { skipDoc: true, expression: `[] | .[] | "foo" + .`, expected: []string{}, }, { skipDoc: true, expression: `[] | .[] | . + "foo"`, expected: []string{}, }, { skipDoc: true, expression: `select(.) | "foo" + "bar"`, expected: []string{ "D0, P[], (!!str)::foobar\n", // jq does not do this :/ - but yq has for quite some time. }, }, { skipDoc: true, document: "apples: 3", expression: `.apples + 3`, expected: []string{ "D0, P[apples], (!!int)::6\n", }, }, { skipDoc: true, document: "apples: 3", expression: `.bobo + 3`, expected: []string{ "D0, P[], (!!int)::3\n", }, }, { skipDoc: true, expression: `select(.) | "cat" + .`, expected: []string{}, }, { skipDoc: true, document: `[]`, expression: `.[] | (.a + "|" + .b)`, expected: []string{}, }, { skipDoc: true, document: `[]`, expression: `.[] | (.a + "|")`, expected: []string{}, }, { skipDoc: true, document: `[]`, expression: `.[] | ("|" + .a)`, expected: []string{}, }, { skipDoc: true, document: `resources: [foo, bar, baz]`, expression: `.missing + .resources | .[]`, expected: []string{ "D0, P[resources 0], (!!str)::foo\n", "D0, P[resources 1], (!!str)::bar\n", "D0, P[resources 2], (!!str)::baz\n", }, }, { skipDoc: true, document: `resources: [foo, bar, baz]`, expression: `. | .missing + .resources | .[]`, expected: []string{ "D0, P[resources 0], (!!str)::foo\n", "D0, P[resources 1], (!!str)::bar\n", "D0, P[resources 2], (!!str)::baz\n", }, }, { skipDoc: true, document: `resources: [foo, bar, baz]`, expression: `. | .missing + .resources`, expected: []string{ "D0, P[resources], (!!seq)::[foo, bar, baz]\n", }, }, { skipDoc: true, document: `resources: [foo, bar, baz]`, expression: `. | .missing + .resources | .[]`, expected: []string{ "D0, P[resources 0], (!!str)::foo\n", "D0, P[resources 1], (!!str)::bar\n", "D0, P[resources 2], (!!str)::baz\n", }, }, { skipDoc: true, document: `[{a: foo, b: bar}, {a: 1, b: 2}]`, expression: ".[] | .a + .b", expected: []string{ "D0, P[0 a], (!!str)::foobar\n", "D0, P[1 a], (!!int)::3\n", }, }, { skipDoc: true, description: "add sequence creates a new sequence", expression: `["a"] as $f | {0:$f + ["b"], 1:$f}`, expected: []string{ "D0, P[], (!!map)::0:\n - a\n - b\n1:\n - a\n", }, }, { skipDoc: true, document: `a: key`, expression: `. += {"key": "b"}`, expected: []string{ "D0, P[], (!!map)::a: key\nkey: b\n", }, }, { skipDoc: true, document: `[[c], [b]]`, expression: `.[] | . += "a"`, expected: []string{ "D0, P[0], (!!seq)::[c, a]\n", "D0, P[1], (!!seq)::[b, a]\n", }, }, { skipDoc: true, document: `{}`, expression: "(.a + .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: `a: 0`, expression: ".a += .b.c", expected: []string{ "D0, P[], (!!map)::a: 0\n", }, }, { description: "Concatenate arrays", document: `{a: [1,2], b: [3,4]}`, expression: `.a + .b`, expected: []string{ "D0, P[a], (!!seq)::[1, 2, 3, 4]\n", }, }, { description: "Concatenate to existing array", subdescription: "Note that the styling of `a` is kept.", document: "a: [1,2]\nb:\n - 3\n - 4", dontFormatInputForDoc: true, expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: [1, 2, 3, 4]\nb:\n - 3\n - 4\n", }, }, { skipDoc: true, expression: `[1] + ([2], [3])`, expected: []string{ "D0, P[], (!!seq)::- 1\n- 2\n", "D0, P[], (!!seq)::- 1\n- 3\n", }, }, { description: "Concatenate null to array", document: `{a: [1,2]}`, expression: `.a + null`, expected: []string{ "D0, P[a], (!!seq)::[1, 2]\n", }, }, { skipDoc: true, description: "Concatenate to empty array", document: `{a: []}`, expression: `.a + "cat"`, expected: []string{ "D0, P[a], (!!seq)::- cat\n", }, }, { description: "Append to existing array", subdescription: "Note that the styling is copied from existing array elements", dontFormatInputForDoc: true, document: `a: ['dog']`, expression: `.a += "cat"`, expected: []string{ "D0, P[], (!!map)::a: ['dog', 'cat']\n", }, }, { description: "Prepend to existing array", document: `a: [dog]`, expression: `.a = ["cat"] + .a`, expected: []string{ "D0, P[], (!!map)::a: [cat, dog]\n", }, }, { skipDoc: true, description: "Concatenate to existing array", subdescription: "does not modify original", document: `{a: ['dog'], b: cat}`, expression: `.a = .a + .b`, expected: []string{ "D0, P[], (!!map)::{a: ['dog', 'cat'], b: cat}\n", }, }, { skipDoc: true, description: "Concatenate to empty array", document: `a: []`, expression: `.a += "cat"`, expected: []string{ "D0, P[], (!!map)::a:\n - cat\n", }, }, { skipDoc: true, description: "Concatenate to existing array", document: `a: [dog]`, expression: `.a += "cat"`, expected: []string{ "D0, P[], (!!map)::a: [dog, cat]\n", }, }, { skipDoc: true, description: "Concatenate to empty object", document: `{a: {}}`, expression: `.a + {"b": "cat"}`, expected: []string{ "D0, P[a], (!!map)::b: cat\n", }, }, { skipDoc: true, description: "Concatenate to existing object", document: `{a: {c: dog}}`, expression: `.a + {"b": "cat"}`, expected: []string{ "D0, P[a], (!!map)::{c: dog, b: cat}\n", }, }, { skipDoc: true, description: "Concatenate to existing object", subdescription: "matches stylig", document: "a:\n c: dog", expression: `.a + {"b": "cat"}`, expected: []string{ "D0, P[a], (!!map)::c: dog\nb: cat\n", }, }, { skipDoc: true, description: "Concatenate to empty object in place", document: `a: {}`, expression: `.a += {"b": "cat"}`, expected: []string{ "D0, P[], (!!map)::a:\n b: cat\n", }, }, { skipDoc: true, description: "Concatenate to existing object in place", document: `a: {c: dog}`, expression: `.a += {"b": "cat"}`, expected: []string{ "D0, P[], (!!map)::a: {c: dog, b: cat}\n", }, }, { description: "Add new object to array", document: `a: [{dog: woof}]`, expression: `.a + {"cat": "meow"}`, expected: []string{ "D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\n", }, }, { description: "Relative append", document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`, expression: `.a[].b += ["mouse"]`, expected: []string{ "D0, P[], (!!map)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n", }, }, { description: "String concatenation", document: `{a: cat, b: meow}`, expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::{a: catmeow, b: meow}\n", }, }, { description: "String concatenation - str + int", skipDoc: true, document: `{a: !cool cat, b: meow}`, expression: `.a + 3`, expected: []string{ "D0, P[a], (!cool)::cat3\n", }, }, { description: "String concatenation - int + str", skipDoc: true, document: `{a: !cool cat, b: meow}`, expression: `3 + .a`, expected: []string{ "D0, P[], (!cool)::3cat\n", }, }, { skipDoc: true, expression: `null + "cat"`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { skipDoc: true, expression: `"cat" + null`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { description: "Number addition - float", subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", document: `{a: 3, b: 4.9}`, expression: `.a = .a + .b`, expected: []string{ "D0, P[], (!!map)::{a: 7.9, b: 4.9}\n", }, }, { description: "Number addition - int", subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", document: `{a: 3, b: 4}`, expression: `.a = .a + .b`, expected: []string{ "D0, P[], (!!map)::{a: 7, b: 4}\n", }, }, { description: "Increment numbers", document: `{a: 3, b: 5}`, expression: `.[] += 1`, expected: []string{ "D0, P[], (!!map)::{a: 4, b: 6}\n", }, }, { description: "Date addition", subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: 2021-01-01T00:00:00Z`, expression: `.a += "3h10m"`, expected: []string{ "D0, P[], (!!map)::a: 2021-01-01T03:10:00Z\n", }, }, { description: "Date addition -date only", skipDoc: true, document: `a: 2021-01-01`, expression: `.a += "24h"`, expected: []string{ "D0, P[], (!!map)::a: 2021-01-02T00:00:00Z\n", }, }, { description: "Date addition - custom format", subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", }, }, { skipDoc: true, description: "Date addition - custom format", subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, expected: []string{ "D0, P[], (!!map)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", }, }, { skipDoc: true, description: "empty add shouldn't add", document: `[]`, expression: `.[] | (.a + "cat")`, expected: []string{}, }, { skipDoc: true, description: "empty add shouldn't add", document: `[]`, expression: `.[] | (.a + "cat" + .b)`, expected: []string{}, }, { skipDoc: true, description: "Add to empty", subdescription: "should behave like null", expression: `.nada + "cat"`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { description: "Add to null", subdescription: "Adding to null simply returns the rhs", expression: `null + "cat"`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { description: "Add maps to shallow merge", subdescription: "Adding objects together shallow merges them. Use `*` to deeply merge.", document: "a: {thing: {name: Astuff, value: x}, a1: cool}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: {thing: {name: Bstuff, legs: 3}, a1: cool, b1: neat}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\n", }, }, { description: "Custom types: that are really strings", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse cat\nb: !goat _meow", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: !horse cat_meow\nb: !goat _meow\n", }, }, { description: "Custom types: that are really numbers", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse 1.2\nb: !goat 2.3", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 3.5\nb: !goat 2.3\n", }, }, { skipDoc: true, document: "a: !horse 2\nb: !goat 2.3", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 4.3\nb: !goat 2.3\n", }, }, { skipDoc: true, document: "a: 2\nb: !goat 2.3", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: 4.3\nb: !goat 2.3\n", }, }, { skipDoc: true, description: "Custom types: that are really ints", document: "a: !horse 2\nb: !goat 3", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 5\nb: !goat 3\n", }, }, { description: "Custom types: that are really arrays", skipDoc: true, subdescription: "when custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse [a]\nb: !goat [b]", expression: `.a += .b`, expected: []string{ "D0, P[], (!!map)::a: !horse [a, b]\nb: !goat [b]\n", }, }, { skipDoc: true, description: "Keep anchors", document: "a: &horse [1]", expression: `.a += 2`, expected: []string{ "D0, P[], (!!map)::a: &horse [1, 2]\n", }, }, { skipDoc: true, description: "Add sequence to map", document: "a: {x: cool}", expression: `.a += [2]`, expectedError: "!!seq () cannot be added to a !!map (a)", }, { skipDoc: true, description: "Add sequence to scalar", document: "a: cool", expression: `.a += [2]`, expectedError: "!!seq () cannot be added to a !!str (a)", }, } func TestAddOperatorScenarios(t *testing.T) { for _, tt := range addOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "add", addOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_alternative.go ================================================ package yqlib func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("alternative") prefs := crossFunctionPreferences{ CalcWhenEmpty: true, Calculation: alternativeFunc, LhsResultValue: func(lhs *CandidateNode) (*CandidateNode, error) { if lhs == nil { return nil, nil } truthy := isTruthyNode(lhs) if truthy { return lhs, nil } return nil, nil }, } return crossFunctionWithPrefs(d, context, expressionNode, prefs) } func alternativeFunc(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if lhs == nil { return rhs, nil } if rhs == nil { return lhs, nil } isTrue := isTruthyNode(lhs) if isTrue { return lhs, nil } return rhs, nil } ================================================ FILE: pkg/yqlib/operator_alternative_test.go ================================================ package yqlib import ( "testing" ) var alternativeOperatorScenarios = []expressionScenario{ { // to match jq - we do not use a readonly clone context on the LHS. skipDoc: true, expression: `.b // .c`, document: `a: bridge`, expected: []string{ "D0, P[c], (!!null)::null\n", }, }, { skipDoc: true, expression: `(.b // "hello") as $x | .`, document: `a: bridge`, expected: []string{ "D0, P[], (!!map)::a: bridge\n", }, }, { skipDoc: true, expression: `.a // .b`, document: `a: 2`, expected: []string{ "D0, P[a], (!!int)::2\n", }, }, { description: "LHS is defined", expression: `.a // "hello"`, document: `{a: bridge}`, expected: []string{ "D0, P[a], (!!str)::bridge\n", }, }, { expression: `select(tag == "seq") // "cat"`, skipDoc: true, document: `a: frog`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { description: "LHS is not defined", expression: `.a // "hello"`, document: `{}`, expected: []string{ "D0, P[], (!!str)::hello\n", }, }, { description: "LHS is null", expression: `.a // "hello"`, document: `{a: ~}`, expected: []string{ "D0, P[], (!!str)::hello\n", }, }, { description: "LHS is false", expression: `.a // "hello"`, document: `{a: false}`, expected: []string{ "D0, P[], (!!str)::hello\n", }, }, { description: "RHS is an expression", expression: `.a // .b`, document: `{a: false, b: cat}`, expected: []string{ "D0, P[b], (!!str)::cat\n", }, }, { skipDoc: true, expression: `false // true`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "Update or create - entity exists", subdescription: "This initialises `a` if it's not present", expression: "(.a // (.a = 0)) += 1", document: `a: 1`, expected: []string{ "D0, P[], (!!map)::a: 2\n", }, }, { description: "Update or create - entity does not exist", subdescription: "This initialises `a` if it's not present", expression: "(.a // (.a = 0)) += 1", document: `b: camel`, expected: []string{ "D0, P[], (!!map)::b: camel\na: 1\n", }, }, } func TestAlternativeOperatorScenarios(t *testing.T) { for _, tt := range alternativeOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "alternative-default-value", alternativeOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_anchors_aliases.go ================================================ package yqlib import ( "container/list" "fmt" ) var showMergeAnchorToSpecWarning = true func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignAlias operator!") aliasName := "" if !expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { aliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("Setting aliasName : %v", candidate.GetKey()) if expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { aliasName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } if aliasName != "" { candidate.Kind = AliasNode candidate.Value = aliasName } } return context, nil } func getAliasOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetAlias operator!") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!str", candidate.Value) results.PushBack(result) } return context.ChildContext(results), nil } func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignAnchor operator!") anchorName := "" if !expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { anchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("Setting anchorName of : %v", candidate.GetKey()) if expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { anchorName = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } candidate.Anchor = anchorName } return context, nil } func getAnchorOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetAnchor operator!") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) anchor := candidate.Anchor result := candidate.CreateReplacement(ScalarNode, "!!str", anchor) results.PushBack(result) } return context.ChildContext(results), nil } func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("ExplodeOperation") for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } for childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() { err = explodeNode(childEl.Value.(*CandidateNode), context) if err != nil { return Context{}, err } } } return context, nil } func fixedReconstructAliasedMap(node *CandidateNode) error { var newContent = []*CandidateNode{} for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] if keyNode.Tag != "!!merge" { // always add in plain nodes // explode both the key and value nodes if err := explodeNode(keyNode, Context{}); err != nil { return err } if err := explodeNode(valueNode, Context{}); err != nil { return err } newContent = append(newContent, keyNode, valueNode) } else { sequence := valueNode if sequence.Kind == AliasNode { sequence = sequence.Alias } if sequence.Kind != SequenceNode { sequence = &CandidateNode{Content: []*CandidateNode{sequence}} } for index := 0; index < len(sequence.Content); index = index + 1 { // for merge anchors, we only set them if the key is not already in node or the newContent mergeNodeSeq := sequence.Content[index] if mergeNodeSeq.Kind == AliasNode { mergeNodeSeq = mergeNodeSeq.Alias } if mergeNodeSeq.Kind != MappingNode { return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", mergeNodeSeq.Tag) } itemsToAdd := mergeNodeSeq.FilterMapContentByKey(func(keyNode *CandidateNode) bool { return getContentValueByKey(node.Content, keyNode.Value) == nil && getContentValueByKey(newContent, keyNode.Value) == nil }) for _, item := range itemsToAdd { // copy to ensure exploding doesn't modify the original node itemCopy := item.Copy() if err := explodeNode(itemCopy, Context{}); err != nil { return err } newContent = append(newContent, itemCopy) } } } } node.Content = newContent return nil } func reconstructAliasedMap(node *CandidateNode, context Context) error { var newContent = list.New() // can I short cut here by prechecking if there's an anchor in the map? // no it needs to recurse in overrideEntry. for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] log.Debugf("traversing %v", keyNode.Value) if keyNode.Value != "<<" { err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent)) if err != nil { return err } } else { if valueNode.Kind == SequenceNode { log.Debugf("an alias merge list!") for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { aliasNode := valueNode.Content[index] err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent)) if err != nil { return err } } } else { log.Debugf("an alias merge!") err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent)) if err != nil { return err } } } } node.Content = make([]*CandidateNode, 0) for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() { node.AddChild(newEl.Value.(*CandidateNode)) } return nil } func explodeNode(node *CandidateNode, context Context) error { log.Debugf("explodeNode - %v", NodeToString(node)) node.Anchor = "" switch node.Kind { case SequenceNode: for index, contentNode := range node.Content { log.Debugf("explodeNode - index %v", index) errorInContent := explodeNode(contentNode, context) if errorInContent != nil { return errorInContent } } return nil case AliasNode: log.Debugf("explodeNode - an alias to %v", NodeToString(node.Alias)) if node.Alias != nil { node.Kind = node.Alias.Kind node.Style = node.Alias.Style node.Tag = node.Alias.Tag node.AddChildren(node.Alias.Content) node.Value = node.Alias.Value node.Alias = nil } log.Debug("now I'm %v", NodeToString(node)) return nil case MappingNode: // //check the map has an alias in it hasAlias := false for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] if keyNode.Value == "<<" { hasAlias = true break } } if hasAlias { if ConfiguredYamlPreferences.FixMergeAnchorToSpec { return fixedReconstructAliasedMap(node) } if showMergeAnchorToSpecWarning { log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025.") showMergeAnchorToSpecWarning = false } // this is a slow op, which is why we want to check before running it. return reconstructAliasedMap(node, context) } // this map has no aliases, but it's kids might for index := 0; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] err := explodeNode(keyNode, context) if err != nil { return err } err = explodeNode(valueNode, context) if err != nil { return err } } return nil default: return nil } } func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newContent Context) error { log.Debug("alias is nil ?") if alias == nil { return nil } log.Debug("alias: %v", NodeToString(alias)) if alias.Kind != MappingNode { return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", alias.Tag) } for index := 0; index < len(alias.Content); index = index + 2 { keyNode := alias.Content[index] log.Debugf("applying alias key %v", keyNode.Value) valueNode := alias.Content[index+1] err := overrideEntry(node, keyNode, valueNode, aliasIndex, newContent) if err != nil { return err } } return nil } func overrideEntry(node *CandidateNode, key *CandidateNode, value *CandidateNode, startIndex int, newContent Context) error { err := explodeNode(value, newContent) if err != nil { return err } for newEl := newContent.MatchingNodes.Front(); newEl != nil; newEl = newEl.Next() { valueEl := newEl.Next() // move forward twice keyNode := newEl.Value.(*CandidateNode) log.Debugf("checking new content %v:%v", keyNode.Value, valueEl.Value.(*CandidateNode).Value) if keyNode.Value == key.Value && keyNode.Alias == nil && key.Alias == nil { log.Debugf("overridign new content") valueEl.Value = value return nil } newEl = valueEl // move forward twice } for index := startIndex + 2; index < len(node.Content); index = index + 2 { keyNode := node.Content[index] if keyNode.Value == key.Value && keyNode.Alias == nil { log.Debugf("content will be overridden at index %v", index) return nil } } err = explodeNode(key, newContent) if err != nil { return err } log.Debugf("adding %v:%v", key.Value, value.Value) newContent.MatchingNodes.PushBack(key) newContent.MatchingNodes.PushBack(value) return nil } ================================================ FILE: pkg/yqlib/operator_anchors_aliases_test.go ================================================ package yqlib import ( "testing" ) var specDocument = `- &CENTRE { x: 1, y: 2 } - &LEFT { x: 0, y: 2 } - &BIG { r: 10 } - &SMALL { r: 1 } ` var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n" var simpleArrayRef = `item_value: &item_value value: true thingOne: name: item_1 <<: *item_value thingTwo: name: item_2 <<: *item_value ` var expectedUpdatedArrayRef = `D0, P[], (!!map)::item_value: &item_value value: true thingOne: name: item_1 value: false thingTwo: name: item_2 !!merge <<: *item_value ` var explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo: a: foo_a thing: foo_thing c: foo_c bar: b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b a: foo_a thing: foo_thing c: foobarList_c foobar: c: foobar_c a: foo_a thing: foobar_thing ` var explodeMergeAnchorsExpected = `D0, P[], (!!map)::foo: a: foo_a thing: foo_thing c: foo_c bar: b: bar_b thing: bar_thing c: bar_c foobarList: b: bar_b thing: foo_thing c: foobarList_c a: foo_a foobar: c: foo_c a: foo_a thing: foobar_thing ` var explodeWhenKeysExistDocument = `objects: - &circle name: circle shape: round - name: ellipse !!merge <<: *circle - !!merge <<: *circle name: egg ` var explodeWhenKeysExistLegacy = `D0, P[], (!!map)::objects: - name: circle shape: round - name: circle shape: round - shape: round name: egg ` var explodeWhenKeysExistExpected = `D0, P[], (!!map)::objects: - name: circle shape: round - name: ellipse shape: round - shape: round name: egg ` var fixedAnchorOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "merge anchor after existing keys", subdescription: "Does not override existing keys - note the name field in the second element is still ellipse.", document: explodeWhenKeysExistDocument, expression: "explode(.)", expected: []string{explodeWhenKeysExistExpected}, }, { description: "FIXED: Explode with merge anchors", subdescription: "Observe that foobarList.b property is still foobarList_b.", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsFixedExpected}, }, { skipDoc: true, document: mergeDocSample, expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", "D0, P[foobarList], (!!map)::{b: foobarList_b, a: foo_a, thing: foo_thing, c: foobarList_c}\n", "D0, P[foobar], (!!map)::{c: foobar_c, a: foo_a, thing: foobar_thing}\n", }, }, { description: "FIXED: Merge multiple maps", subdescription: "Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.", document: specDocument + "- << : [ *CENTRE, *BIG ]\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"}, }, { description: "FIXED: Override", subdescription: "Taken from https://yaml.org/type/merge.html. Same values as legacy, but with the correct key order.", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\ny: 2\nx: 1\n"}, }, { description: "Exploding inline merge anchor", // subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: {c: *b}}`, expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, }, { skipDoc: true, description: "Exploding inline merge anchor in sequence", subdescription: "`<<` map must be exploded, otherwise `c: *b` will become invalid", document: `{a: {b: &b 42}, <<: [{c: *b}]}`, expression: `explode(.) | sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: {b: 42}, c: 42}\n", }, }, { skipDoc: true, description: "Exploding merge anchor should not explode neighbours", subdescription: "b must not be exploded, as `r: *a` will become invalid", document: `{b: &b {a: &a 42}, r: *a, c: {<<: *b}}`, expression: `explode(.c)`, expected: []string{ "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", }, }, { skipDoc: true, description: "Exploding sequence merge anchor should not explode neighbours", subdescription: "b must not be exploded, as `r: *a` will become invalid", document: `{b: &b {a: &a 42}, r: *a, c: {<<: [*b]}}`, expression: `explode(.c)`, expected: []string{ "D0, P[], (!!map)::{b: &b {a: &a 42}, r: *a, c: {a: 42}}\n", }, }, { skipDoc: true, description: "Merge anchor with inline map", document: `{<<: {a: 42}}`, expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: 42}\n", }, }, { skipDoc: true, description: "Merge anchor with sequence with inline map", document: `{<<: [{a: 42}]}`, expression: `explode(.)`, expected: []string{ "D0, P[], (!!map)::{a: 42}\n", }, }, { skipDoc: true, description: "Merge anchor with aliased sequence with inline map", document: `{s: &s [{a: 42}], m: {<<: *s}}`, expression: `.m | explode(.)`, expected: []string{ "D0, P[m], (!!map)::{a: 42}\n", }, }, { skipDoc: true, description: "deleting after explode", document: "x: 37\na: &a\n b: 42\n<<: *a", expression: `explode(.) | del(.x)`, expected: []string{ "D0, P[], (!!map)::a:\n b: 42\nb: 42\n", }, }, } var badAnchorOperatorScenarios = []expressionScenario{ { skipDoc: true, // incorrect overrides description: "LEGACY: merge anchor after existing keys", document: explodeWhenKeysExistDocument, expression: "explode(.)", expected: []string{explodeWhenKeysExistLegacy}, }, { description: "LEGACY: Explode with merge anchors", // incorrect overrides subdescription: "Caution: this is for when --yaml-fix-merge-anchor-to-spec=false; it's not to YAML spec because the merge anchors incorrectly override the object values (foobarList.b is set to bar_b when it should still be foobarList_b). Flag will default to true in late 2025", document: mergeDocSample, expression: `explode(.)`, expected: []string{explodeMergeAnchorsExpected}, }, { skipDoc: true, document: mergeDocSample, // incorrect overrides expression: `.foo* | explode(.) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foo* | explode(explode(.)) | (. style="flow")`, expected: []string{ "D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n", "D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n", "D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n", }, }, { description: "LEGACY: Merge multiple maps", subdescription: "see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.", document: specDocument + "- << : [ *CENTRE, *BIG ]\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, }, { description: "LEGACY: Override", subdescription: "see https://yaml.org/type/merge.html. This has the correct data, but the wrong key order; set --yaml-fix-merge-anchor-to-spec=true to fix the key order.", document: specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n x: 1\n", expression: ".[4] | explode(.)", expected: []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"}, }, } var anchorOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "merge anchor to alias alias", document: "b: &b 10\na: &a { k: *b }\nc:\n <<: [*a]", expression: "explode(.)", expected: []string{"D0, P[], (!!map)::b: 10\na: {k: 10}\nc:\n k: 10\n"}, }, { skipDoc: true, description: "merge anchor not map", document: "a: &a\n - 0\nc:\n <<: [*a]\n", expectedError: "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing !!seq", expression: "explode(.)", }, { description: "Merge one map", subdescription: "see https://yaml.org/type/merge.html", document: specDocument + "- << : *CENTRE\n r: 10\n", expression: ".[4] | explode(.)", expected: []string{expectedSpecResult}, }, { description: "Get anchor", document: `a: &billyBob cat`, expression: `.a | anchor`, expected: []string{ "D0, P[a], (!!str)::billyBob\n", }, }, { description: "Set anchor", document: `a: cat`, expression: `.a anchor = "foobar"`, expected: []string{ "D0, P[], (!!map)::a: &foobar cat\n", }, }, { description: "Set anchor relatively using assign-update", document: `a: {b: cat}`, expression: `.a anchor |= .b`, expected: []string{ "D0, P[], (!!map)::a: &cat {b: cat}\n", }, }, { skipDoc: true, document: `a: {c: cat}`, expression: `.a anchor |= .b`, expected: []string{ "D0, P[], (!!map)::a: {c: cat}\n", }, }, { skipDoc: true, document: `a: {c: cat}`, expression: `.a anchor = .b`, expected: []string{ "D0, P[], (!!map)::a: {c: cat}\n", }, }, { description: "Get alias", document: `{b: &billyBob meow, a: *billyBob}`, expression: `.a | alias`, expected: []string{ "D0, P[a], (!!str)::billyBob\n", }, }, { description: "Set alias", document: `{b: &meow purr, a: cat}`, expression: `.a alias = "meow"`, expected: []string{ "D0, P[], (!!map)::{b: &meow purr, a: *meow}\n", }, }, { description: "Set alias to blank does nothing", document: `{b: &meow purr, a: cat}`, expression: `.a alias = ""`, expected: []string{ "D0, P[], (!!map)::{b: &meow purr, a: cat}\n", }, }, { skipDoc: true, document: `{b: &meow purr, a: cat}`, expression: `.a alias = .c`, expected: []string{ "D0, P[], (!!map)::{b: &meow purr, a: cat}\n", }, }, { skipDoc: true, document: `{b: &meow purr, a: cat}`, expression: `.a alias |= .c`, expected: []string{ "D0, P[], (!!map)::{b: &meow purr, a: cat}\n", }, }, { description: "Set alias relatively using assign-update", document: `{b: &meow purr, a: {f: meow}}`, expression: `.a alias |= .f`, expected: []string{ "D0, P[], (!!map)::{b: &meow purr, a: *meow}\n", }, }, { description: "Dont explode alias and anchor - check alias parent", skipDoc: true, document: `{a: &a [1], b: *a}`, expression: `.b[]`, expected: []string{ "D0, P[a 0], (!!int)::1\n", }, }, { description: "Explode alias and anchor - check alias parent", skipDoc: true, document: `{a: &a cat, b: *a}`, expression: `explode(.) | .b`, expected: []string{ "D0, P[b], (!!str)::cat\n", }, }, { description: "Explode splat", skipDoc: true, document: `{a: &a cat, b: *a}`, expression: `explode(.)[]`, expected: []string{ "D0, P[a], (!!str)::cat\n", "D0, P[b], (!!str)::cat\n", }, }, { description: "Explode alias and anchor - check original parent", skipDoc: true, document: `{a: &a cat, b: *a}`, expression: `explode(.) | .a`, expected: []string{ "D0, P[a], (!!str)::cat\n", }, }, { description: "Explode alias and anchor", document: `{f : {a: &a cat, b: *a}}`, expression: `explode(.f)`, expected: []string{ "D0, P[], (!!map)::{f: {a: cat, b: cat}}\n", }, }, { description: "Explode with no aliases or anchors", document: `a: mike`, expression: `explode(.a)`, expected: []string{ "D0, P[], (!!map)::a: mike\n", }, }, { description: "Explode with alias keys", subdescription: "No space between alias", skipDoc: true, document: `{f : {a: &a cat, *a: b}}`, expression: `explode(.f)`, expected: []string{ "D0, P[], (!!map)::{f: {a: cat, cat: b}}\n", }, skipForGoccy: true, // can't handle no space between alias }, { description: "Explode with alias keys", document: `{f : {a: &a cat, *a : b}}`, expression: `explode(.f)`, expected: []string{ "D0, P[], (!!map)::{f: {a: cat, cat: b}}\n", }, }, { skipDoc: true, document: `{f : {a: &a cat, b: &b {foo: *a}, *a: *b}}`, expression: `explode(.f)`, expected: []string{ "D0, P[], (!!map)::{f: {a: cat, b: {foo: cat}, cat: {foo: cat}}}\n", }, }, { description: "Dereference and update a field", subdescription: "Use explode with multiply to dereference an object", document: simpleArrayRef, expression: `.thingOne |= (explode(.) | sort_keys(.)) * {"value": false}`, expected: []string{expectedUpdatedArrayRef}, }, { skipDoc: true, description: "Duplicate keys", subdescription: "outside merge anchor", document: `{a: 1, a: 2}`, expression: `explode(.)`, expected: []string{ // {a: 2} would also be fine "D0, P[], (!!map)::{a: 1, a: 2}\n", }, }, { skipDoc: true, description: "!!str << should not be treated as merge anchor", document: `{!!str <<: {a: 37}}`, expression: `explode(.).a`, expected: []string{ "D0, P[a], (!!null)::null\n", }, }, } func TestAnchorAliasOperatorScenarios(t *testing.T) { for _, tt := range append(anchorOperatorScenarios, badAnchorOperatorScenarios...) { testScenario(t, &tt) } documentOperatorScenarios(t, "anchor-and-alias-operators", append(anchorOperatorScenarios, badAnchorOperatorScenarios...)) } func TestAnchorAliasOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true for _, tt := range append(fixedAnchorOperatorScenarios, anchorOperatorScenarios...) { testScenario(t, &tt) } for i, tt := range fixedAnchorOperatorScenarios { fixedAnchorOperatorScenarios[i].subdescription = "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour (flag will default to true in late 2025).\n" + tt.subdescription } appendOperatorDocumentScenario(t, "anchor-and-alias-operators", fixedAnchorOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false } ================================================ FILE: pkg/yqlib/operator_array_to_map_test.go ================================================ package yqlib import ( "testing" ) var arrayToMapScenarios = []expressionScenario{ { description: "Simple example", document: `cool: [null, null, hello]`, expression: `.cool |= array_to_map`, expected: []string{ "D0, P[], (!!map)::cool:\n 2: hello\n", }, }, } func TestArrayToMapScenarios(t *testing.T) { for _, tt := range arrayToMapScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "array-to-map", arrayToMapScenarios) } ================================================ FILE: pkg/yqlib/operator_assign.go ================================================ package yqlib type assignPreferences struct { DontOverWriteAnchor bool OnlyWriteNull bool ClobberCustomTags bool } func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation { return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if !prefs.OnlyWriteNull || lhs.Tag == "!!null" { lhs.UpdateFrom(rhs, prefs) } return lhs, nil } } // they way *= (multipleAssign) is handled, we set the multiplePrefs // on the node, not assignPrefs. Long story. func getAssignPreferences(preferences interface{}) assignPreferences { prefs := assignPreferences{} switch typedPref := preferences.(type) { case assignPreferences: prefs = typedPref case multiplyPreferences: prefs = typedPref.AssignPrefs } return prefs } func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } prefs := getAssignPreferences(expressionNode.Operation.Preferences) log.Debug("assignUpdateOperator prefs: %v", prefs) if !expressionNode.Operation.UpdateAssign { // this works because we already ran against LHS with an editable context. _, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc(prefs), false) return context, err } //traverse backwards through the context - // like delete, we need to run against the children first. // (e.g. consider when running with expression '.. |= [.]' - we need // to wrap the children first for el := lhs.MatchingNodes.Back(); el != nil; el = el.Prev() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } // grab the first value first := rhs.MatchingNodes.Front() if first != nil { rhsCandidate := first.Value.(*CandidateNode) candidate.UpdateFrom(rhsCandidate, prefs) } } return context, nil } // does not update content or values func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("getting lhs matching nodes for update") lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } // grab the first value first := rhs.MatchingNodes.Front() if first != nil { prefs := assignPreferences{} if expressionNode.Operation.Preferences != nil { prefs = expressionNode.Operation.Preferences.(assignPreferences) } if !prefs.OnlyWriteNull || candidate.Tag == "!!null" { candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs) } } } return context, nil } ================================================ FILE: pkg/yqlib/operator_assign_test.go ================================================ package yqlib import ( "testing" ) var mergeAnchorAssign = `a: &a x: OriginalValue b: <<: *a` var assignOperatorScenarios = []expressionScenario{ { description: "Create yaml file", expression: `.a.b = "cat" | .x = "frog"`, expected: []string{ "D0, P[], ()::a:\n b: cat\nx: frog\n", }, }, { description: "Create yaml file", document: "a: {b: 3}", expression: `.a |= .`, skipDoc: true, expected: []string{ "D0, P[], (!!map)::a: {b: 3}\n", }, }, { skipDoc: true, document: "{}", expression: `.a |= .b`, expected: []string{ "D0, P[], (!!map)::a: null\n", }, }, { skipDoc: true, document: mergeAnchorAssign, expression: `.c = .b | .a.x = "ModifiedValue" | explode(.)`, expected: []string{ "D0, P[], (!!map)::a:\n x: ModifiedValue\nb:\n x: ModifiedValue\nc:\n x: ModifiedValue\n", }, }, { skipDoc: true, document: "{}", expression: `.a = .b`, expected: []string{ "D0, P[], (!!map)::a: null\n", }, }, { skipDoc: true, description: "self reference", document: "a: cat", expression: `.a = [.a]`, expected: []string{ "D0, P[], (!!map)::a:\n - cat\n", }, }, { skipDoc: true, description: "change to number when old value is valid number", document: `a: "3"`, expression: `.a = 3`, expected: []string{ "D0, P[], (!!map)::a: 3\n", }, }, { skipDoc: true, description: "change to bool when old value is valid bool", document: `a: "true"`, expression: `.a = true`, expected: []string{ "D0, P[], (!!map)::a: true\n", }, }, { skipDoc: true, description: "update custom tag string, dont clobber style", document: `a: !cat "meow"`, expression: `.a = "woof"`, expected: []string{ "D0, P[], (!!map)::a: !cat \"woof\"\n", }, }, { description: "Update node to be the child value", document: `{a: {b: {g: foof}}}`, expression: `.a |= .b`, expected: []string{ "D0, P[], (!!map)::{a: {g: foof}}\n", }, }, { description: "Double elements in an array", document: `[1,2,3]`, expression: `.[] |= . * 2`, expected: []string{ "D0, P[], (!!seq)::[2, 4, 6]\n", }, }, { description: "Update node from another file", subdescription: "Note this will also work when the second file is a scalar (string/number)", document: `{a: apples}`, document2: "{b: bob}", expression: `select(fileIndex==0).a = select(fileIndex==1) | select(fileIndex==0)`, expected: []string{ "D0, P[], (!!map)::{a: {b: bob}}\n", }, }, { description: "Update node to be the sibling value", document: `{a: {b: child}, b: sibling}`, expression: `.a = .b`, expected: []string{ "D0, P[], (!!map)::{a: sibling, b: sibling}\n", }, }, { description: "Updated multiple paths", document: `{a: fieldA, b: fieldB, c: fieldC}`, expression: `(.a, .c) = "potato"`, expected: []string{ "D0, P[], (!!map)::{a: potato, b: fieldB, c: potato}\n", }, }, { description: "Update string value", document: `{a: {b: apple}}`, expression: `.a.b = "frog"`, expected: []string{ "D0, P[], (!!map)::{a: {b: frog}}\n", }, }, { description: "Update string value via |=", subdescription: "Note there is no difference between `=` and `|=` when the RHS is a scalar", document: `{a: {b: apple}}`, expression: `.a.b |= "frog"`, expected: []string{ "D0, P[], (!!map)::{a: {b: frog}}\n", }, }, { skipDoc: true, document: `{a: {b: apple}}`, expression: `.a.b | (. |= "frog")`, expected: []string{ "D0, P[a b], (!!str)::frog\n", }, }, { skipDoc: true, document: `{a: {b: apple}}`, expression: `.a.b |= 5`, expected: []string{ "D0, P[], (!!map)::{a: {b: 5}}\n", }, }, { skipDoc: true, document: `{a: {b: apple}}`, expression: `.a.b |= 3.142`, expected: []string{ "D0, P[], (!!map)::{a: {b: 3.142}}\n", }, }, { description: "Update deeply selected results", subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.", document: `{a: {b: apple, c: cactus}}`, expression: `(.a[] | select(. == "apple")) = "frog"`, expected: []string{ "D0, P[], (!!map)::{a: {b: frog, c: cactus}}\n", }, }, { skipDoc: true, document: `{a: {b: apple, c: cactus}}`, expression: `(.a.[] | select(. == "apple")) = "frog"`, expected: []string{ "D0, P[], (!!map)::{a: {b: frog, c: cactus}}\n", }, }, { description: "Update array values", document: `[candy, apple, sandy]`, expression: `(.[] | select(. == "*andy")) = "bogs"`, expected: []string{ "D0, P[], (!!seq)::[bogs, apple, bogs]\n", }, }, { description: "Update empty object", dontFormatInputForDoc: true, document: `{}`, expression: `.a.b |= "bogs"`, expected: []string{ "D0, P[], (!!map)::a:\n b: bogs\n", }, }, { description: "Update node value that has an anchor", subdescription: "Anchor will remain", dontFormatInputForDoc: true, document: `a: &cool cat`, expression: `.a = "dog"`, expected: []string{ "D0, P[], (!!map)::a: &cool dog\n", }, }, { description: "Update empty object and array", dontFormatInputForDoc: true, document: `{}`, expression: `.a.b.[0] |= "bogs"`, expected: []string{ "D0, P[], (!!map)::a:\n b:\n - bogs\n", }, }, { skipDoc: true, document: `{}`, expression: `.a.b.[1].c |= "bogs"`, expected: []string{ "D0, P[], (!!map)::a:\n b:\n - null\n - c: bogs\n", }, }, { description: "Custom types are maintained by default", document: "a: !cat meow\nb: !dog woof", expression: `.a = .b`, expected: []string{ "D0, P[], (!!map)::a: !cat woof\nb: !dog woof\n", }, }, { description: "Custom types: clobber", subdescription: "Use the `c` option to clobber custom tags", document: "a: !cat meow\nb: !dog woof", expression: `.a =c .b`, expected: []string{ "D0, P[], (!!map)::a: !dog woof\nb: !dog woof\n", }, }, } func TestAssignOperatorScenarios(t *testing.T) { for _, tt := range assignOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "assign-update", assignOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_booleans.go ================================================ package yqlib import ( "container/list" "fmt" "strings" ) func isTruthyNode(node *CandidateNode) bool { if node == nil { return false } if node.Tag == "!!null" { return false } if node.Kind == ScalarNode && node.Tag == "!!bool" { // yes/y/true/on return (strings.EqualFold(node.Value, "y") || strings.EqualFold(node.Value, "yes") || strings.EqualFold(node.Value, "on") || strings.EqualFold(node.Value, "true")) } return true } func getOwner(lhs *CandidateNode, rhs *CandidateNode) *CandidateNode { owner := lhs if lhs == nil && rhs == nil { owner = &CandidateNode{} } else if lhs == nil { owner = rhs } return owner } func returnRhsTruthy(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { owner := getOwner(lhs, rhs) rhsBool := isTruthyNode(rhs) return createBooleanCandidate(owner, rhsBool), nil } func returnLHSWhen(targetBool bool) func(lhs *CandidateNode) (*CandidateNode, error) { return func(lhs *CandidateNode) (*CandidateNode, error) { var err error var lhsBool bool if lhsBool = isTruthyNode(lhs); lhsBool != targetBool { return nil, err } owner := &CandidateNode{} if lhs != nil { owner = lhs } return createBooleanCandidate(owner, targetBool), nil } } func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *CandidateNode) (bool, error) { for _, node := range sequenceNode.Content { if expressionNode != nil { //need to evaluate the expression against the node rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(node), expressionNode) if err != nil { return false, err } if rhs.MatchingNodes.Len() > 0 { node = rhs.MatchingNodes.Front().Value.(*CandidateNode) } else { // no results found, ignore this entry continue } } if isTruthyNode(node) == wantBool { return true, nil } } return false, nil } func allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return Context{}, fmt.Errorf("all only supports arrays, was %v", candidate.Tag) } booleanResult, err := findBoolean(false, d, context, expressionNode.RHS, candidate) if err != nil { return Context{}, err } result := createBooleanCandidate(candidate, !booleanResult) results.PushBack(result) } return context.ChildContext(results), nil } func anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return Context{}, fmt.Errorf("any only supports arrays, was %v", candidate.Tag) } booleanResult, err := findBoolean(true, d, context, expressionNode.RHS, candidate) if err != nil { return Context{}, err } result := createBooleanCandidate(candidate, booleanResult) results.PushBack(result) } return context.ChildContext(results), nil } func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { prefs := crossFunctionPreferences{ CalcWhenEmpty: true, Calculation: returnRhsTruthy, LhsResultValue: returnLHSWhen(true), } return crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs) } func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { prefs := crossFunctionPreferences{ CalcWhenEmpty: true, Calculation: returnRhsTruthy, LhsResultValue: returnLHSWhen(false), } return crossFunctionWithPrefs(d, context.ReadOnlyClone(), expressionNode, prefs) } func notOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("notOperation") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debug("notOperation checking %v", candidate) truthy := isTruthyNode(candidate) result := createBooleanCandidate(candidate, !truthy) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_booleans_test.go ================================================ package yqlib import ( "testing" ) var booleanOperatorScenarios = []expressionScenario{ { description: "`or` example", expression: `true or false`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "\"yes\" and \"no\" are strings", subdescription: "In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/", document: `[yes, no]`, expression: `.[] | tag`, expected: []string{ "D0, P[0], (!!str)::!!str\n", "D0, P[1], (!!str)::!!str\n", }, }, { skipDoc: true, document: "b: hi", expression: `.a or .c`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: "b: false", expression: `.b or .c`, expected: []string{ "D0, P[b], (!!bool)::false\n", }, }, { skipDoc: true, document: "b: hi", expression: `select(.a or .b)`, expected: []string{ "D0, P[], (!!map)::b: hi\n", }, }, { skipDoc: true, document: "b: hi", expression: `select((.a and .b) | not)`, expected: []string{ "D0, P[], (!!map)::b: hi\n", }, }, { skipDoc: true, description: "And should not run 2nd arg if first is false", expression: `false and test(3)`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, description: "Or should not run 2nd arg if first is true", expression: `true or test(3)`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "`and` example", expression: `true and false`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { document: "[{a: bird, b: dog}, {a: frog, b: bird}, {a: cat, b: fly}]", description: "Matching nodes with select, equals and or", expression: `[.[] | select(.a == "cat" or .b == "dog")]`, expected: []string{ "D0, P[], (!!seq)::- {a: bird, b: dog}\n- {a: cat, b: fly}\n", }, }, { description: "`any` returns true if any boolean in a given array is true", document: `[false, true]`, expression: "any", expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "`any` returns false for an empty array", document: `[]`, expression: "any", expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "`any_c` returns true if any element in the array is true for the given condition.", document: "a: [rad, awesome]\nb: [meh, whatever]", expression: `.[] |= any_c(. == "awesome")`, expected: []string{ "D0, P[], (!!map)::a: true\nb: false\n", }, }, { skipDoc: true, document: `[{pet: cat}]`, expression: `any_c(.name == "harry") as $c | .`, expected: []string{ "D0, P[], (!!seq)::[{pet: cat}]\n", }, }, { skipDoc: true, document: `[{pet: cat}]`, expression: `any_c(.name == "harry") as $c | $c`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: `[{pet: cat}]`, expression: `all_c(.name == "harry") as $c | $c`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: `[false, false]`, expression: "any", expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "`all` returns true if all booleans in a given array are true", document: `[true, true]`, expression: "all", expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { skipDoc: true, document: `[false, true]`, expression: "all", expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "`all` returns true for an empty array", document: `[]`, expression: "all", expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "`all_c` returns true if all elements in the array are true for the given condition.", document: "a: [rad, awesome]\nb: [meh, 12]", expression: `.[] |= all_c(tag == "!!str")`, expected: []string{ "D0, P[], (!!map)::a: true\nb: false\n", }, }, { skipDoc: true, expression: `false or false`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: `{a: true, b: false}`, expression: `.[] or (false, true)`, expected: []string{ "D0, P[a], (!!bool)::true\n", "D0, P[b], (!!bool)::false\n", "D0, P[b], (!!bool)::true\n", }, }, { skipDoc: true, document: `{a: true, b: false}`, expression: `.[] and (false, true)`, expected: []string{ "D0, P[a], (!!bool)::false\n", "D0, P[a], (!!bool)::true\n", "D0, P[b], (!!bool)::false\n", }, }, { skipDoc: true, document: `{}`, expression: `(.a.b or .c) as $x | .`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: `{}`, expression: `(.a.b and .c) as $x | .`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { description: "Not true is false", expression: `true | not`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Not false is true", expression: `false | not`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "String values considered to be true", expression: `"cat" | not`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Empty string value considered to be true", expression: `"" | not`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Numbers are considered to be true", expression: `1 | not`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Zero is considered to be true", expression: `0 | not`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Null is considered to be false", expression: `~ | not`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, } func TestBooleanOperatorScenarios(t *testing.T) { for _, tt := range booleanOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "boolean-operators", booleanOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_collect.go ================================================ package yqlib import ( "container/list" ) func collectTogether(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*CandidateNode, error) { collectedNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode) if err != nil { return nil, err } for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { resultC := result.Value.(*CandidateNode) log.Debugf("found this: %v", NodeToString(resultC)) collectedNode.AddChild(resultC) } } return collectedNode, nil } func collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("collectOperation") if context.MatchingNodes.Len() == 0 { log.Debugf("nothing to collect") node := &CandidateNode{Kind: SequenceNode, Tag: "!!seq", Value: "[]"} return context.SingleChildContext(node), nil } var evaluateAllTogether = true for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether if !evaluateAllTogether { break } } if evaluateAllTogether { log.Debugf("collect together") collectedNode, err := collectTogether(d, context, expressionNode.RHS) if err != nil { return Context{}, err } return context.SingleChildContext(collectedNode), nil } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) collectCandidate := candidate.CreateReplacement(SequenceNode, "!!seq", "") log.Debugf("collect rhs: %v", expressionNode.RHS.Operation.toString()) collectExpResults, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() { resultC := result.Value.(*CandidateNode) log.Debugf("found this: %v", NodeToString(resultC)) collectCandidate.AddChild(resultC) } log.Debugf("done collect rhs: %v", expressionNode.RHS.Operation.toString()) results.PushBack(collectCandidate) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_collect_object.go ================================================ package yqlib import ( "container/list" "fmt" ) /* [Mike: cat, Bob: dog] [Thing: rabbit, peter: sam] ==> cross multiply {Mike: cat, Thing: rabbit} {Mike: cat, peter: sam} ... */ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, _ *ExpressionNode) (Context, error) { log.Debugf("collectObjectOperation") context := originalContext.WritableClone() if context.MatchingNodes.Len() == 0 { candidate := &CandidateNode{Kind: MappingNode, Tag: "!!map", Value: "{}"} log.Debugf("collectObjectOperation - starting with empty map") return context.SingleChildContext(candidate), nil } first := context.MatchingNodes.Front().Value.(*CandidateNode) var rotated = make([]*list.List, len(first.Content)) for i := 0; i < len(first.Content); i++ { rotated[i] = list.New() } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidateNode := el.Value.(*CandidateNode) if len(candidateNode.Content) < len(first.Content) { return Context{}, fmt.Errorf("CollectObject: mismatching node sizes; are you creating a map with mismatching key value pairs?") } for i := 0; i < len(first.Content); i++ { log.Debugf("rotate[%v] = %v", i, NodeToString(candidateNode.Content[i])) log.Debugf("children:\n%v", NodeContentToString(candidateNode.Content[i], 0)) rotated[i].PushBack(candidateNode.Content[i]) } } log.Debugf("collectObjectOperation, length of rotated is %v", len(rotated)) newObject := list.New() for i := 0; i < len(first.Content); i++ { additions, err := collect(d, context.ChildContext(list.New()), rotated[i]) if err != nil { return Context{}, err } // we should reset the parents and keys of these top level nodes, // as they are new for el := additions.MatchingNodes.Front(); el != nil; el = el.Next() { addition := el.Value.(*CandidateNode) additionCopy := addition.Copy() additionCopy.SetParent(nil) additionCopy.Key = nil log.Debugf("collectObjectOperation, adding result %v", NodeToString(additionCopy)) newObject.PushBack(additionCopy) } } return context.ChildContext(newObject), nil } func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) (Context, error) { if remainingMatches.Len() == 0 { return context, nil } candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode) log.Debugf("collectObjectOperation - collect %v", NodeToString(candidate)) splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}) if err != nil { return Context{}, err } if context.MatchingNodes.Len() == 0 { log.Debugf("collectObjectOperation - collect context is empty, next") return collect(d, splatted, remainingMatches) } newAgg := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { aggCandidate := el.Value.(*CandidateNode) for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() { splatCandidate := splatEl.Value.(*CandidateNode) log.Debugf("collectObjectOperation; splatCandidate: %v", NodeToString(splatCandidate)) newCandidate := aggCandidate.Copy() log.Debugf("collectObjectOperation; aggCandidate: %v", NodeToString(aggCandidate)) newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate) if err != nil { return Context{}, err } newAgg.PushBack(newCandidate) } } return collect(d, context.ChildContext(newAgg), remainingMatches) } ================================================ FILE: pkg/yqlib/operator_collect_object_test.go ================================================ package yqlib import ( "testing" ) var collectObjectOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: `{"name": "mike"} | .name`, expected: []string{ "D0, P[name], (!!str)::mike\n", }, }, { skipDoc: true, expression: `{"c": "a", "b", "d"}`, expectedError: "CollectObject: mismatching node sizes; are you creating a map with mismatching key value pairs?", }, { skipDoc: true, expression: `{"person": {"names": ["mike"]}} | .person.names[0]`, expected: []string{ "D0, P[person names 0], (!!str)::mike\n", }, }, { skipDoc: true, document: `[{name: cat}, {name: dog}]`, expression: `.[] | {.name: "great"}`, expected: []string{ "D0, P[], (!!map)::cat: great\n", "D0, P[], (!!map)::dog: great\n", }, }, { description: "collect splat", skipDoc: true, document: `[{name: cat}, {name: dog}]`, expression: `.[] | {.name: "great"}[]`, expected: []string{ "D0, P[cat], (!!str)::great\n", "D0, P[dog], (!!str)::great\n", }, }, { skipDoc: true, expression: `({} + {}) | (.b = 3)`, expected: []string{ "D0, P[], (!!map)::b: 3\n", }, }, { skipDoc: true, document: "a: []", expression: `.a += [{"key": "att2", "value": "val2"}]`, expected: []string{ "D0, P[], (!!map)::a:\n - key: att2\n value: val2\n", }, }, { skipDoc: true, document: "", expression: `.a += {"key": "att2", "value": "val2"}`, expected: []string{ "D0, P[], ()::a:\n key: att2\n value: val2\n", }, }, { skipDoc: true, document: "", expression: `.a += [0]`, expected: []string{ "D0, P[], ()::a:\n - 0\n", }, }, { description: `Collect empty object`, document: ``, expression: `{}`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { description: `Wrap (prefix) existing object`, document: "{name: Mike}\n", expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", }, }, { skipDoc: true, description: "Two documents", document: "{name: Mike}\n", document2: "{name: Bob}\n", expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", "D0, P[], (!!map)::wrap: {name: Bob}\n", }, }, { skipDoc: true, description: "two embedded documents", document: "{name: Mike}\n---\n{name: Bob}", expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", "D1, P[], (!!map)::wrap: {name: Bob}\n", }, }, { skipDoc: true, document: `{name: Mike, age: 32}`, expression: `{.name: .age}`, expected: []string{ "D0, P[], (!!map)::Mike: 32\n", }, }, { description: `Using splat to create multiple objects`, document: `{name: Mike, pets: [cat, dog]}`, expression: `{.name: .pets.[]}`, expected: []string{ "D0, P[], (!!map)::Mike: cat\n", "D0, P[], (!!map)::Mike: dog\n", }, }, { description: `Working with multiple documents`, dontFormatInputForDoc: false, document: "{name: Mike, pets: [cat, dog]}\n---\n{name: Rosey, pets: [monkey, sheep]}", expression: `{.name: .pets.[]}`, expected: []string{ "D0, P[], (!!map)::Mike: cat\n", "D0, P[], (!!map)::Mike: dog\n", "D1, P[], (!!map)::Rosey: monkey\n", "D1, P[], (!!map)::Rosey: sheep\n", }, }, { skipDoc: true, document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, expression: `{.name: .pets.[], "f":.food.[]}`, expected: []string{ "D0, P[], (!!map)::Mike: cat\nf: hotdog\n", "D0, P[], (!!map)::Mike: cat\nf: burger\n", "D0, P[], (!!map)::Mike: dog\nf: hotdog\n", "D0, P[], (!!map)::Mike: dog\nf: burger\n", }, }, { skipDoc: true, document: "name: Mike\npets:\n cows:\n - apl\n - bba", document2: "name: Rosey\npets:\n sheep:\n - frog\n - meow", expression: `{"a":.name, "b":.pets}`, expected: []string{ "D0, P[], (!!map)::a: Mike\nb:\n cows:\n - apl\n - bba\n", "D0, P[], (!!map)::a: Rosey\nb:\n sheep:\n - frog\n - meow\n", }, }, { description: "Creating yaml from scratch", document: ``, expression: `{"wrap": "frog"}`, expected: []string{ "D0, P[], (!!map)::wrap: frog\n", }, }, { skipDoc: true, expression: `{"wrap": "frog", "bing": "bong"}`, expected: []string{ "D0, P[], (!!map)::wrap: frog\nbing: bong\n", }, }, { skipDoc: true, document: `{name: Mike}`, expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", }, }, { skipDoc: true, document: `{name: Mike}`, expression: `{"wrap": {"further": .}} | (.. style= "flow")`, expected: []string{ "D0, P[], (!!map)::{wrap: {further: {name: Mike}}}\n", }, }, { description: "Creating yaml from scratch with multiple objects", expression: `(.a.b = "foo") | (.d.e = "bar")`, expected: []string{ "D0, P[], ()::a:\n b: foo\nd:\n e: bar\n", }, }, } func TestCollectObjectOperatorScenarios(t *testing.T) { for _, tt := range collectObjectOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "create-collect-into-object", collectObjectOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_collect_test.go ================================================ package yqlib import ( "testing" ) var collectOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: `["x", "y"] | .[1]`, expected: []string{ "D0, P[1], (!!str)::y\n", }, }, { skipDoc: true, document: ``, expression: `.a += [0]`, expected: []string{ "D0, P[], ()::a:\n - 0\n", }, }, { skipDoc: true, expression: `[1,2,3]`, expected: []string{ "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", }, }, { skipDoc: true, description: "update in collect", expression: `[.a = "cat"]`, expected: []string{ "D0, P[], (!!seq)::- a: cat\n", }, }, { description: "Collect empty", document: ``, expression: `[]`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: "{a: apple}\n---\n{b: frog}", expression: `[.]`, expected: []string{ "D0, P[], (!!seq)::- {a: apple}\n- {b: frog}\n", }, }, { description: "with comments", skipDoc: true, document: "# abc\n[{a: apple}]\n\n# xyz\n", expression: `[.[]]`, expected: []string{ "D0, P[], (!!seq)::- {a: apple}\n", }, }, { skipDoc: true, document: ``, expression: `[3]`, expected: []string{ "D0, P[], (!!seq)::- 3\n", }, }, { description: "Collect single", document: ``, expression: `["cat"]`, expected: []string{ "D0, P[], (!!seq)::- cat\n", }, }, { document: ``, skipDoc: true, expression: `[true]`, expected: []string{ "D0, P[], (!!seq)::- true\n", }, }, { description: "Collect many", document: `{a: cat, b: dog}`, expression: `[.a, .b]`, expected: []string{ "D0, P[], (!!seq)::- cat\n- dog\n", }, }, { document: ``, skipDoc: true, expression: `collect(1)`, expected: []string{ "D0, P[], (!!seq)::- 1\n", }, }, { document: `[1,2,3]`, skipDoc: true, expression: `[.[]]`, expected: []string{ "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", }, }, { skipDoc: true, expression: `[1,2][]`, expected: []string{ "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", }, }, { document: `a: {b: [1,2,3]}`, expression: `[.a.b.[]]`, skipDoc: true, expected: []string{ "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", }, }, { skipDoc: true, document: `[{name: cat, thing: bor}, {name: dog}]`, expression: `.[] | [.name]`, expected: []string{ "D0, P[0], (!!seq)::- cat\n", "D0, P[1], (!!seq)::- dog\n", }, }, } func TestCollectOperatorScenarios(t *testing.T) { for _, tt := range collectOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "collect-into-array", collectOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_column.go ================================================ package yqlib import ( "container/list" "fmt" ) func columnOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("columnOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.Column)) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_column_test.go ================================================ package yqlib import ( "testing" ) var columnOperatorScenarios = []expressionScenario{ { description: "Returns column of _value_ node", document: "a: cat\nb: bob", expression: `.b | column`, expected: []string{ "D0, P[b], (!!int)::4\n", }, }, { description: "Returns column of _key_ node", subdescription: "Pipe through the key operator to get the column of the key", document: "a: cat\nb: bob", expression: `.b | key | column`, expected: []string{ "D0, P[b], (!!int)::1\n", }, }, { description: "First column is 1", document: "a: cat", expression: `.a | key | column`, expected: []string{ "D0, P[a], (!!int)::1\n", }, }, { description: "No column data is 0", expression: `{"a": "new entry"} | column`, expected: []string{ "D0, P[], (!!int)::0\n", }, }, } func TestColumnOperatorScenarios(t *testing.T) { for _, tt := range columnOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "column", columnOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_comments.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "regexp" ) type commentOpPreferences struct { LineComment bool HeadComment bool FootComment bool } func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignComments operator!") lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } preferences := expressionNode.Operation.Preferences.(commentOpPreferences) comment := "" if !expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { comment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } log.Debugf("AssignComments comment is %v", comment) for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("AssignComments lhs %v", NodeToString(candidate)) if expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { comment = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } log.Debugf("Setting comment of : %v", candidate.GetKey()) if preferences.LineComment { log.Debugf("Setting line comment of : %v to %v", candidate.GetKey(), comment) candidate.LineComment = comment } if preferences.HeadComment { candidate.HeadComment = comment candidate.LeadingContent = "" // clobber the leading content, if there was any. } if preferences.FootComment { candidate.FootComment = comment } } return context, nil } func getCommentsOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { preferences := expressionNode.Operation.Preferences.(commentOpPreferences) var startCommentCharacterRegExp = regexp.MustCompile(`^# `) var subsequentCommentCharacterRegExp = regexp.MustCompile(`\n# `) log.Debugf("GetComments operator!") var results = list.New() yamlPrefs := ConfiguredYamlPreferences.Copy() yamlPrefs.PrintDocSeparators = false yamlPrefs.UnwrapScalar = false yamlPrefs.ColorsEnabled = false for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) comment := "" if preferences.LineComment { log.Debugf("Reading line comment of : %v to %v", candidate.GetKey(), candidate.LineComment) comment = candidate.LineComment } else if preferences.HeadComment && candidate.LeadingContent != "" { var chompRegexp = regexp.MustCompile(`\n$`) var output bytes.Buffer var writer = bufio.NewWriter(&output) var encoder = NewYamlEncoder(yamlPrefs) if err := encoder.PrintLeadingContent(writer, candidate.LeadingContent); err != nil { return Context{}, err } if err := writer.Flush(); err != nil { return Context{}, err } comment = output.String() comment = chompRegexp.ReplaceAllString(comment, "") } else if preferences.HeadComment { comment = candidate.HeadComment } else if preferences.FootComment { comment = candidate.FootComment } comment = startCommentCharacterRegExp.ReplaceAllString(comment, "") comment = subsequentCommentCharacterRegExp.ReplaceAllString(comment, "\n") result := candidate.CreateReplacement(ScalarNode, "!!str", comment) if candidate.IsMapKey { result.IsMapKey = false result.Key = candidate } results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_comments_test.go ================================================ package yqlib import ( "testing" ) var expectedWhereIsMyCommentMapKey = `D0, P[], (!!seq)::- p: "" isKey: false hc: "" lc: "" fc: "" - p: hello isKey: true hc: "" lc: hello-world-comment fc: "" - p: hello isKey: false hc: "" lc: "" fc: "" - p: hello.message isKey: true hc: "" lc: "" fc: "" - p: hello.message isKey: false hc: "" lc: "" fc: "" ` var expectedWhereIsMyCommentArray = `D0, P[], (!!seq)::- p: "" isKey: false hc: "" lc: "" fc: "" - p: name isKey: true hc: "" lc: "" fc: "" - p: name isKey: false hc: "" lc: "" fc: "" - p: name.0 isKey: false hc: under-name-comment lc: "" fc: "" ` var commentOperatorScenarios = []expressionScenario{ { description: "Set line comment", subdescription: "Set the comment on the key node for more reliability (see below).", document: `a: cat`, expression: `.a line_comment="single"`, expected: []string{ "D0, P[], (!!map)::a: cat # single\n", }, }, { description: "Set line comment of a maps/arrays", subdescription: "For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.", document: "a:\n b: things", expression: `(.a | key) line_comment="single"`, expected: []string{ "D0, P[], (!!map)::a: # single\n b: things\n", }, }, { skipDoc: true, document: "a: cat\nb: dog", expression: `.a line_comment=.b`, expected: []string{ "D0, P[], (!!map)::a: cat # dog\nb: dog\n", }, }, { skipDoc: true, document: "a: cat\n---\na: dog", expression: `.a line_comment |= documentIndex`, expected: []string{ "D0, P[], (!!map)::a: cat # 0\n", "D1, P[], (!!map)::a: dog # 1\n", }, }, { description: "Use update assign to perform relative updates", document: "a: cat\nb: dog", expression: `.. line_comment |= .`, expected: []string{ "D0, P[], (!!map)::a: cat # cat\nb: dog # dog\n", }, }, { skipDoc: true, document: "a: cat\nb: dog", expression: `.. comments |= .`, expected: []string{ "D0, P[], (!!map)::a: cat # cat\n# cat\n\n# cat\nb: dog # dog\n# dog\n\n# dog\n", }, }, { description: "Where is the comment - map key example", subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'hello-world-comment' is actually on the 'hello' key", document: "hello: # hello-world-comment\n message: world", expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, expected: []string{ expectedWhereIsMyCommentMapKey, }, }, { description: "Retrieve comment - map key example", subdescription: "From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment", document: "hello: # hello-world-comment\n message: world", expression: `.hello | key | line_comment`, expected: []string{ "D0, P[hello], (!!str)::hello-world-comment\n", }, }, { description: "Where is the comment - array example", subdescription: "The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).\nFrom this, you can see the 'under-name-comment' is actually on the first child", document: "name:\n # under-name-comment\n - first-array-child", expression: `[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]`, expected: []string{ expectedWhereIsMyCommentArray, }, }, { description: "Retrieve comment - array example", subdescription: "From the previous example, we know that the comment is on the first child as a headComment", document: "name:\n # under-name-comment\n - first-array-child", expression: `.name[0] | headComment`, expected: []string{ "D0, P[name 0], (!!str)::under-name-comment\n", }, }, { description: "Set head comment", document: `a: cat`, expression: `. head_comment="single"`, expected: []string{ "D0, P[], (!!map)::# single\na: cat\n", }, }, { description: "Set head comment of a map entry", document: "f: foo\na:\n b: cat", expression: `(.a | key) head_comment="single"`, expected: []string{ "D0, P[], (!!map)::f: foo\n# single\na:\n b: cat\n", }, }, { description: "Set foot comment, using an expression", document: `a: cat`, expression: `. foot_comment=.a`, expected: []string{ "D0, P[], (!!map)::a: cat\n# cat\n", }, }, { skipDoc: true, description: "Set foot comment, using an expression", document: "a: cat\n\n# hi", expression: `. foot_comment=""`, expected: []string{ "D0, P[], (!!map)::a: cat\n", }, }, { skipDoc: true, document: `a: cat`, expression: `. foot_comment=.b.d`, expected: []string{ "D0, P[], (!!map)::a: cat\n", }, }, { skipDoc: true, document: `a: cat`, expression: `. foot_comment|=.b.d`, expected: []string{ "D0, P[], (!!map)::a: cat\n", }, }, { description: "Remove comment", document: "a: cat # comment\nb: dog # leave this", expression: `.a line_comment=""`, expected: []string{ "D0, P[], (!!map)::a: cat\nb: dog # leave this\n", }, }, { description: "Remove (strip) all comments", subdescription: "Note the use of `...` to ensure key nodes are included.", document: "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment", expression: `... comments=""`, expected: []string{ "D0, P[], (!!map)::a: cat\nb:\n", }, }, { description: "Get line comment", document: "# welcome!\n\na: cat # meow\n\n# have a great day", expression: `.a | line_comment`, expected: []string{ "D0, P[a], (!!str)::meow\n", }, }, { description: "Get head comment", dontFormatInputForDoc: true, document: "# welcome!\n\na: cat # meow\n\n# have a great day", expression: `. | head_comment`, expected: []string{ "D0, P[], (!!str)::welcome!\n\n", }, }, { skipDoc: true, description: "strip trailing comment recurse all", document: "a: cat\n\n# haha", expression: `... comments= ""`, expected: []string{ "D0, P[], (!!map)::a: cat\n", }, }, { skipDoc: true, description: "strip trailing comment recurse values", document: "a: cat\n\n# haha", expression: `.. comments= ""`, expected: []string{ "D0, P[], (!!map)::a: cat\n", }, }, { description: "Head comment with document split", dontFormatInputForDoc: true, document: "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day", expression: `head_comment`, expected: []string{ "D0, P[], (!!str)::welcome!\nbob\n", }, }, { description: "Get foot comment", dontFormatInputForDoc: true, document: "# welcome!\n\na: cat # meow\n\n# have a great day\n# no really", expression: `. | foot_comment`, expected: []string{ "D0, P[], (!!str)::have a great day\nno really\n", }, }, { description: "leading spaces", skipDoc: true, document: " # hi", expression: `.`, expected: []string{ "D0, P[], (!!null):: # hi\n", }, }, { description: "string spaces", skipDoc: true, document: "# hi\ncat\n", expression: `.`, expected: []string{ "D0, P[], (!!str)::# hi\ncat\n", }, }, { description: "leading spaces with new line", skipDoc: true, document: " # hi\n", expression: `.`, expected: []string{ "D0, P[], (!!null):: # hi\n", }, }, { description: "directive", skipDoc: true, document: "%YAML 1.1\n# hi\n", expression: `.`, expected: []string{ "D0, P[], (!!null)::%YAML 1.1\n# hi\n", }, }, } func TestCommentOperatorScenarios(t *testing.T) { for _, tt := range commentOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "comment-operators", commentOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_compare.go ================================================ package yqlib import ( "container/list" "fmt" "strconv" ) type compareTypePref struct { OrEqual bool Greater bool } func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("compareOperator") prefs := expressionNode.Operation.Preferences.(compareTypePref) return crossFunction(d, context, expressionNode, compare(prefs), true) } func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { return func(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { log.Debugf("compare cross function") if lhs == nil && rhs == nil { owner := &CandidateNode{} return createBooleanCandidate(owner, prefs.OrEqual), nil } else if lhs == nil { log.Debugf("lhs nil, but rhs is not") return createBooleanCandidate(rhs, false), nil } else if rhs == nil { log.Debugf("rhs nil, but rhs is not") return createBooleanCandidate(lhs, false), nil } switch lhs.Kind { case MappingNode: return nil, fmt.Errorf("maps not yet supported for comparison") case SequenceNode: return nil, fmt.Errorf("arrays not yet supported for comparison") default: if rhs.Kind != ScalarNode { return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Tag, rhs.GetNicePath(), lhs.Tag) } target := lhs.CopyWithoutContent() boolV, err := compareScalars(context, prefs, lhs, rhs) return createBooleanCandidate(target, boolV), err } } } func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) { lhsTime, err := parseDateTime(layout, lhs.Value) if err != nil { return false, err } rhsTime, err := parseDateTime(layout, rhs.Value) if err != nil { return false, err } if prefs.OrEqual && lhsTime.Equal(rhsTime) { return true, nil } if prefs.Greater { return lhsTime.After(rhsTime), nil } return lhsTime.Before(rhsTime), nil } func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) { lhsTag := lhs.guessTagFromCustomType() rhsTag := rhs.guessTagFromCustomType() isDateTime := lhs.Tag == "!!timestamp" // if the lhs is a string, it might be a timestamp in a custom format. if lhsTag == "!!str" { _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) isDateTime = err == nil } if isDateTime { return compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs) } else if lhsTag == "!!int" && rhsTag == "!!int" { _, lhsNum, err := parseInt64(lhs.Value) if err != nil { return false, err } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { return false, err } if prefs.OrEqual && lhsNum == rhsNum { return true, nil } if prefs.Greater { return lhsNum > rhsNum, nil } return lhsNum < rhsNum, nil } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return false, err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return false, err } if prefs.OrEqual && lhsNum == rhsNum { return true, nil } if prefs.Greater { return lhsNum > rhsNum, nil } return lhsNum < rhsNum, nil } else if lhsTag == "!!str" && rhsTag == "!!str" { if prefs.OrEqual && lhs.Value == rhs.Value { return true, nil } if prefs.Greater { return lhs.Value > rhs.Value, nil } return lhs.Value < rhs.Value, nil } else if lhsTag == "!!null" && rhsTag == "!!null" && prefs.OrEqual { return true, nil } else if lhsTag == "!!null" || rhsTag == "!!null" { return false, nil } return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag) } func superlativeByComparison(d *dataTreeNavigator, context Context, prefs compareTypePref) (Context, error) { fn := compare(prefs) var results = list.New() for seq := context.MatchingNodes.Front(); seq != nil; seq = seq.Next() { splatted, err := splat(context.SingleChildContext(seq.Value.(*CandidateNode)), traversePreferences{}) if err != nil { return Context{}, err } result := splatted.MatchingNodes.Front() if result != nil { for el := result.Next(); el != nil; el = el.Next() { cmp, err := fn(d, context, el.Value.(*CandidateNode), result.Value.(*CandidateNode)) if err != nil { return Context{}, err } if isTruthyNode(cmp) { result = el } } results.PushBack(result.Value) } } return context.ChildContext(results), nil } func minOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debug(("Min")) return superlativeByComparison(d, context, compareTypePref{Greater: false}) } func maxOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debug(("Max")) return superlativeByComparison(d, context, compareTypePref{Greater: true}) } ================================================ FILE: pkg/yqlib/operator_contains.go ================================================ package yqlib import ( "fmt" "strings" ) func containsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { return crossFunction(d, context.ReadOnlyClone(), expressionNode, containsWithNodes, false) } func containsArrayElement(array *CandidateNode, item *CandidateNode) (bool, error) { for index := 0; index < len(array.Content); index = index + 1 { containedInArray, err := contains(array.Content[index], item) if err != nil { return false, err } if containedInArray { return true, nil } } return false, nil } func containsArray(lhs *CandidateNode, rhs *CandidateNode) (bool, error) { if rhs.Kind != SequenceNode { return containsArrayElement(lhs, rhs) } for index := 0; index < len(rhs.Content); index = index + 1 { itemInArray, err := containsArrayElement(lhs, rhs.Content[index]) if err != nil { return false, err } if !itemInArray { return false, nil } } return true, nil } func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) { if rhs.Kind != MappingNode { return false, nil } for index := 0; index < len(rhs.Content); index = index + 2 { rhsKey := rhs.Content[index] rhsValue := rhs.Content[index+1] log.Debugf("Looking for %v in the lhs", rhsKey.Value) lhsKeyIndex := findInArray(lhs, rhsKey) log.Debugf("index is %v", lhsKeyIndex) if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 { return false, nil } lhsValue := lhs.Content[lhsKeyIndex+1] log.Debugf("lhsValue is %v", lhsValue.Value) itemInArray, err := contains(lhsValue, rhsValue) log.Debugf("rhsValue is %v", rhsValue.Value) if err != nil { return false, err } if !itemInArray { return false, nil } } return true, nil } func containsScalars(lhs *CandidateNode, rhs *CandidateNode) (bool, error) { if lhs.Tag == "!!str" { return strings.Contains(lhs.Value, rhs.Value), nil } return lhs.Value == rhs.Value, nil } func contains(lhs *CandidateNode, rhs *CandidateNode) (bool, error) { switch lhs.Kind { case MappingNode: return containsObject(lhs, rhs) case SequenceNode: return containsArray(lhs, rhs) case ScalarNode: if rhs.Kind != ScalarNode || lhs.Tag != rhs.Tag { return false, nil } if lhs.Tag == "!!null" { return rhs.Tag == "!!null", nil } return containsScalars(lhs, rhs) } return false, fmt.Errorf("%v not yet supported for contains", lhs.Tag) } func containsWithNodes(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if lhs.Kind != rhs.Kind { return nil, fmt.Errorf("%v cannot check contained in %v", rhs.Tag, lhs.Tag) } result, err := contains(lhs, rhs) if err != nil { return nil, err } return createBooleanCandidate(lhs, result), nil } ================================================ FILE: pkg/yqlib/operator_contains_test.go ================================================ package yqlib import "testing" var containsOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: `null | contains(~)`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { skipDoc: true, expression: `3 | contains(3)`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { skipDoc: true, expression: `3 | contains(32)`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Array contains array", subdescription: "Array is equal or subset of", document: `["foobar", "foobaz", "blarp"]`, expression: `contains(["baz", "bar"])`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "Array has a subset array", subdescription: "Subtract the superset array from the subset, if there's anything left, it's not a subset", document: `["foobar", "foobaz", "blarp"]`, expression: `["baz", "bar"] - . | length == 0`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, expression: `["dog", "cat", "giraffe"] | contains(["camel"])`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Object included in array", document: `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`, expression: `contains({"bar": [{"barp": 12}]})`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "Object not included in array", document: `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`, expression: `contains({"foo": 12, "bar": [{"barp": 15}]})`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "String contains substring", document: `"foobar"`, expression: `contains("bar")`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "String equals string", document: `"meow"`, expression: `contains("meow")`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, } func TestContainsOperatorScenarios(t *testing.T) { for _, tt := range containsOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "contains", containsOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_create_map.go ================================================ package yqlib import ( "container/list" ) func createMapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("createMapOperation") //each matchingNodes entry should turn into a sequence of keys to create. //then collect object should do a cross function of the same index sequence for all matches. sequences := list.New() if context.MatchingNodes.Len() > 0 { for matchingNodeEl := context.MatchingNodes.Front(); matchingNodeEl != nil; matchingNodeEl = matchingNodeEl.Next() { matchingNode := matchingNodeEl.Value.(*CandidateNode) sequenceNode, err := sequenceFor(d, context, matchingNode, expressionNode) if err != nil { return Context{}, err } sequences.PushBack(sequenceNode) } } else { sequenceNode, err := sequenceFor(d, context, nil, expressionNode) if err != nil { return Context{}, err } sequences.PushBack(sequenceNode) } node := listToNodeSeq(sequences) return context.SingleChildContext(node), nil } func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateNode, expressionNode *ExpressionNode) (*CandidateNode, error) { var document uint var filename string var fileIndex int var matches = list.New() if matchingNode != nil { document = matchingNode.GetDocument() filename = matchingNode.GetFilename() fileIndex = matchingNode.GetFileIndex() matches.PushBack(matchingNode) } log.Debugf("**********sequenceFor %v", NodeToString(matchingNode)) mapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode, func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { node := &CandidateNode{Kind: MappingNode, Tag: "!!map"} log.Debugf("**********adding key %v and value %v", NodeToString(lhs), NodeToString(rhs)) node.AddKeyValueChild(lhs, rhs) node.document = document node.fileIndex = fileIndex node.filename = filename return node, nil }, false) if err != nil { return nil, err } innerList := listToNodeSeq(mapPairs.MatchingNodes) innerList.Style = FlowStyle innerList.document = document innerList.fileIndex = fileIndex innerList.filename = filename return innerList, nil } // NOTE: here the document index gets dropped so we // no longer know where the node originates from. func listToNodeSeq(list *list.List) *CandidateNode { node := CandidateNode{Kind: SequenceNode, Tag: "!!seq"} for entry := list.Front(); entry != nil; entry = entry.Next() { entryCandidate := entry.Value.(*CandidateNode) log.Debugf("Collecting %v into sequence", NodeToString(entryCandidate)) node.AddChild(entryCandidate) } return &node } ================================================ FILE: pkg/yqlib/operator_create_map_test.go ================================================ package yqlib import ( "testing" ) var createMapOperatorScenarios = []expressionScenario{ { document: ``, expression: `"frog": "jumps"`, expected: []string{ "D0, P[], (!!seq)::- [{frog: jumps}]\n", }, }, { skipDoc: true, description: "sets key properly", expression: `("frog": "jumps") | .[0][0] | .frog`, expected: []string{ "D0, P[0 0 frog], (!!str)::jumps\n", }, }, { skipDoc: true, description: "sets key properly on map", expression: `{"frog": "jumps"} | .frog`, expected: []string{ "D0, P[frog], (!!str)::jumps\n", }, }, { document: `{name: Mike, pets: [cat, dog]}`, expression: `(.name: .pets.[]) | .[0][0] | ..`, expected: []string{ "D0, P[0 0], (!!map)::Mike: cat\n", "D0, P[0 0 Mike], (!!str)::cat\n", }, }, { description: "check path of nested child", document: "pets:\n cows: value", expression: `("b":.pets) | .[0][0] | .b.cows`, expected: []string{ "D0, P[0 0 b cows], (!!str)::value\n", }, }, { document: `{name: Mike, age: 32}`, expression: `.name: .age`, expected: []string{ "D0, P[], (!!seq)::- [{Mike: 32}]\n", }, }, { document: `{name: Mike, pets: [cat, dog]}`, expression: `.name: .pets.[]`, expected: []string{ "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", }, }, { document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`, expression: `.name: .pets.[], "f":.food.[]`, expected: []string{ "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n", "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n", }, }, { document: "{name: Mike, pets: [cat, dog], food: [hotdog, burger]}\n---\n{name: Fred, pets: [mouse], food: [pizza, onion, apple]}", expression: `.name: .pets.[], "f":.food.[]`, expected: []string{ "D0, P[], (!!seq)::- [{Mike: cat}, {Mike: dog}]\n- [{Fred: mouse}]\n", "D0, P[], (!!seq)::- [{f: hotdog}, {f: burger}]\n- [{f: pizza}, {f: onion}, {f: apple}]\n", }, }, { document: `{name: Mike, pets: {cows: [apl, bba]}}`, expression: `"a":.name, "b":.pets`, expected: []string{ "D0, P[], (!!seq)::- [{a: Mike}]\n", "D0, P[], (!!seq)::- [{b: {cows: [apl, bba]}}]\n", }, }, { document: `{name: Mike}`, expression: `"wrap": .`, expected: []string{ "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n", }, }, { document: "{name: Mike}\n---\n{name: Bob}", expression: `"wrap": .`, expected: []string{ "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", }, }, { document: "{name: Mike}\n---\n{name: Bob}", expression: `"wrap": ., .name: "great"`, expected: []string{ "D0, P[], (!!seq)::- [{wrap: {name: Mike}}]\n- [{wrap: {name: Bob}}]\n", "D0, P[], (!!seq)::- [{Mike: great}]\n- [{Bob: great}]\n", }, }, } func TestCreateMapOperatorScenarios(t *testing.T) { for _, tt := range createMapOperatorScenarios { testScenario(t, &tt) } } ================================================ FILE: pkg/yqlib/operator_datetime.go ================================================ package yqlib import ( "container/list" "errors" "fmt" "strconv" "time" ) func getStringParameter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) { result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode) if err != nil { return "", err } else if result.MatchingNodes.Len() == 0 { return "", fmt.Errorf("could not find %v for format_time", parameterName) } return result.MatchingNodes.Front().Value.(*CandidateNode).Value, nil } func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType { layout, err := getStringParameter("layout", d, context, expressionNode.RHS.LHS) if err != nil { return Context{}, fmt.Errorf("could not get date time format: %w", err) } context.SetDateTimeLayout(layout) return d.GetMatchingNodes(context, expressionNode.RHS.RHS) } return Context{}, errors.New(`must provide a date time format string and an expression, e.g. with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; )`) } // for unit tests var Now = time.Now func nowOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { node := &CandidateNode{ Tag: "!!timestamp", Kind: ScalarNode, Value: Now().Format(time.RFC3339), } return context.SingleChildContext(node), nil } func parseDateTime(layout string, datestring string) (time.Time, error) { parsedTime, err := time.Parse(layout, datestring) if err != nil && layout == time.RFC3339 { // try parsing the date time with only the date return time.Parse("2006-01-02", datestring) } return parsedTime, err } func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { format, err := getStringParameter("format", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() if err != nil { return Context{}, err } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) parsedTime, err := parseDateTime(layout, candidate.Value) if err != nil { return Context{}, fmt.Errorf("could not parse datetime of [%v]: %w", candidate.GetNicePath(), err) } formattedTimeStr := parsedTime.Format(format) node, errorReading := parseSnippet(formattedTimeStr) if errorReading != nil { log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading) node = &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: formattedTimeStr, } } node.Parent = candidate.Parent node.Key = candidate.Key results.PushBack(node) } return context.ChildContext(results), nil } func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { timezoneStr, err := getStringParameter("timezone", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() if err != nil { return Context{}, err } var results = list.New() timezone, err := time.LoadLocation(timezoneStr) if err != nil { return Context{}, fmt.Errorf("could not load tz [%v]: %w", timezoneStr, err) } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) parsedTime, err := parseDateTime(layout, candidate.Value) if err != nil { return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err) } tzTime := parsedTime.In(timezone) results.PushBack(candidate.CreateReplacement(ScalarNode, candidate.Tag, tzTime.Format(layout))) } return context.ChildContext(results), nil } func parseUnixTime(unixTime string) (time.Time, error) { seconds, err := strconv.ParseFloat(unixTime, 64) if err != nil { return time.Now(), err } return time.UnixMilli(int64(seconds * 1000)), nil } func fromUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) actualTag := candidate.guessTagFromCustomType() if actualTag != "!!int" && actualTag != "!!float" { return Context{}, fmt.Errorf("from_unix only works on numbers, found %v instead", candidate.Tag) } parsedTime, err := parseUnixTime(candidate.Value) if err != nil { return Context{}, err } node := candidate.CreateReplacement(ScalarNode, "!!timestamp", parsedTime.Format(time.RFC3339)) results.PushBack(node) } return context.ChildContext(results), nil } func toUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { layout := context.GetDateTimeLayout() var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) parsedTime, err := parseDateTime(layout, candidate.Value) if err != nil { return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err) } results.PushBack(candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", parsedTime.Unix()))) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_datetime_test.go ================================================ package yqlib import ( "testing" ) var dateTimeOperatorScenarios = []expressionScenario{ { description: "Format: from standard RFC3339 format", subdescription: "Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.", document: `a: 2001-12-15T02:59:43.1Z`, expression: `.a |= format_datetime("Monday, 02-Jan-06 at 3:04PM")`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:59AM\n", }, }, { description: "Format: from custom date time", subdescription: "Use with_dtf to set a custom datetime format for parsing.", document: `a: Saturday, 15-Dec-01 at 2:59AM`, expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM"; format_datetime("2006-01-02"))`, expected: []string{ "D0, P[], (!!map)::a: 2001-12-15\n", }, }, { description: "Format: get the day of the week", document: `a: 2001-12-15`, expression: `.a | format_datetime("Monday")`, expected: []string{ "D0, P[a], (!!str)::Saturday\n", }, }, { description: "Now", document: "a: cool", expression: `.updated = now`, expected: []string{ "D0, P[], (!!map)::a: cool\nupdated: 2021-05-19T01:02:03Z\n", }, }, { description: "From Unix", subdescription: "Converts from unix time. Note, you don't have to pipe through the tz operator :)", expression: `1675301929 | from_unix | tz("UTC")`, expected: []string{ "D0, P[], (!!timestamp)::2023-02-02T01:38:49Z\n", }, }, { description: "To Unix", subdescription: "Converts to unix time", expression: `now | to_unix`, expected: []string{ "D0, P[], (!!int)::1621386123\n", }, }, { description: "Timezone: from standard RFC3339 format", subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.", document: "a: cool", expression: `.updated = (now | tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (!!map)::a: cool\nupdated: 2021-05-19T11:02:03+10:00\n", }, }, { description: "Timezone: with custom format", subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, }, { description: "Add and tz custom format", subdescription: "Specify standard IANA Time Zone format or 'utc', 'local'", document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, }, { description: "Date addition", document: `a: 2021-01-01T00:00:00Z`, expression: `.a += "3h10m"`, expected: []string{ "D0, P[], (!!map)::a: 2021-01-01T03:10:00Z\n", }, }, { description: "Date subtraction", subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.", document: `a: 2021-01-01T03:10:00Z`, expression: `.a -= "3h10m"`, expected: []string{ "D0, P[], (!!map)::a: 2021-01-01T00:00:00Z\n", }, }, { description: "Date addition - custom format", document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a += "3h1m")`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", }, }, { description: "Date script with custom format", subdescription: "You can embed full expressions in with_dtf if needed.", document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST"; .a = (.a + "3h1m" | tz("Australia/Perth")))`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:00PM AWST\n", }, }, { description: "allow comma", skipDoc: true, document: "a: Saturday, 15-Dec-01 at 2:59AM GMT", expression: `.a |= with_dtf("Monday, 02-Jan-06 at 3:04PM MST", tz("Australia/Sydney"))`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 1:59PM AEDT\n", }, }, } func TestDatetimeOperatorScenarios(t *testing.T) { for _, tt := range dateTimeOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "datetime", dateTimeOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_delete.go ================================================ package yqlib import ( "container/list" "fmt" ) func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } //need to iterate backwards to ensure correct indices when deleting multiple for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() { candidate := el.Value.(*CandidateNode) if candidate.Parent == nil { // must be a top level thing, delete it return removeFromContext(context, candidate) } log.Debugf("processing deletion of candidate %v", NodeToString(candidate)) parentNode := candidate.Parent candidatePath := candidate.GetPath() childPath := candidatePath[len(candidatePath)-1] switch parentNode.Kind { case MappingNode: deleteFromMap(candidate.Parent, childPath) case SequenceNode: deleteFromArray(candidate.Parent, childPath) default: return Context{}, fmt.Errorf("cannot delete nodes from parent of tag %v", parentNode.Tag) } } return context, nil } func removeFromContext(context Context, candidate *CandidateNode) (Context, error) { newResults := list.New() for item := context.MatchingNodes.Front(); item != nil; item = item.Next() { nodeInContext := item.Value.(*CandidateNode) if nodeInContext != candidate { newResults.PushBack(nodeInContext) } else { log.Info("Need to delete this %v", NodeToString(nodeInContext)) } } return context.ChildContext(newResults), nil } func deleteFromMap(node *CandidateNode, childPath interface{}) { log.Debug("deleteFromMap") contents := node.Content newContents := make([]*CandidateNode, 0) for index := 0; index < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] shouldDelete := key.Value == childPath log.Debugf("shouldDelete %v? %v == %v = %v", NodeToString(value), key.Value, childPath, shouldDelete) if !shouldDelete { newContents = append(newContents, key, value) } } node.Content = newContents } func deleteFromArray(node *CandidateNode, childPath interface{}) { log.Debug("deleteFromArray") contents := node.Content newContents := make([]*CandidateNode, 0) for index := 0; index < len(contents); index = index + 1 { value := contents[index] shouldDelete := fmt.Sprintf("%v", index) == fmt.Sprintf("%v", childPath) if !shouldDelete { value.Key.Value = fmt.Sprintf("%v", len(newContents)) newContents = append(newContents, value) } } node.Content = newContents } ================================================ FILE: pkg/yqlib/operator_delete_test.go ================================================ package yqlib import ( "testing" ) var deleteOperatorScenarios = []expressionScenario{ { description: "Delete entry in map", document: `{a: cat, b: dog}`, expression: `del(.b)`, expected: []string{ "D0, P[], (!!map)::{a: cat}\n", }, }, { description: "Delete nested entry in map", document: `{a: {a1: fred, a2: frood}}`, expression: `del(.a.a1)`, expected: []string{ "D0, P[], (!!map)::{a: {a2: frood}}\n", }, }, { skipDoc: true, document: `{a: {a1: fred, a2: frood}}`, expression: `.a | del(.a1)`, expected: []string{ "D0, P[a], (!!map)::{a2: frood}\n", }, }, { skipDoc: true, description: "delete whole document", document2: `a: slow`, document: `a: fast`, expression: `del(select(.a == "fast"))`, expected: []string{ "D0, P[], (!!map)::a: slow\n", }, }, { skipDoc: true, document: `a: [1,2,3]`, expression: `.a | del(.[1])`, expected: []string{ "D0, P[a], (!!seq)::[1, 3]\n", }, }, { skipDoc: true, document: `[0, {a: cat, b: dog}]`, expression: `.[1] | del(.a)`, expected: []string{ "D0, P[1], (!!map)::{b: dog}\n", }, }, { skipDoc: true, document: `[{a: cat, b: dog}]`, expression: `.[0] | del(.a)`, expected: []string{ "D0, P[0], (!!map)::{b: dog}\n", }, }, { skipDoc: true, document: `[{a: {b: thing, c: frog}}]`, expression: `.[0].a | del(.b)`, expected: []string{ "D0, P[0 a], (!!map)::{c: frog}\n", }, }, { skipDoc: true, document: `[{a: {b: thing, c: frog}}]`, expression: `.[0] | del(.a.b)`, expected: []string{ "D0, P[0], (!!map)::{a: {c: frog}}\n", }, }, { skipDoc: true, document: `{a: [0, {b: thing, c: frog}]}`, expression: `.a[1] | del(.b)`, expected: []string{ "D0, P[a 1], (!!map)::{c: frog}\n", }, }, { skipDoc: true, document: `{a: [0, {b: thing, c: frog}]}`, expression: `.a | del(.[1].b)`, expected: []string{ "D0, P[a], (!!seq)::[0, {c: frog}]\n", }, }, { skipDoc: true, document: `{a: {a1: fred, a2: frood}}`, expression: `del(.. | select(.=="frood"))`, expected: []string{ "D0, P[], (!!map)::{a: {a1: fred}}\n", }, }, { description: "Delete entry in array", document: `[1,2,3]`, expression: `del(.[1])`, expected: []string{ "D0, P[], (!!seq)::[1, 3]\n", }, }, { skipDoc: true, document: `a: [1,2,3]`, expression: `del(.a[])`, expected: []string{ "D0, P[], (!!map)::a: []\n", }, }, { skipDoc: true, description: "Delete entry appended to an array", document: `[1,2]`, expression: `. += [3] | del(.[2])`, expected: []string{ "D0, P[], (!!seq)::[1, 2]\n", }, }, { skipDoc: true, description: "Delete entry after sorting an array", document: `[3,2,1]`, expression: `sort | del(.[2])`, expected: []string{ "D0, P[], (!!seq)::[1, 2]\n", }, }, { skipDoc: true, description: "Delete entry after reversing an array", document: `[1,2,3]`, expression: `reverse | del(.[2])`, expected: []string{ "D0, P[], (!!seq)::[3, 2]\n", }, }, { skipDoc: true, description: "Delete entry after shuffling an array", document: `[1,2,3]`, expression: `shuffle | del(.[2])`, expected: []string{ "D0, P[], (!!seq)::[3, 1]\n", }, }, { skipDoc: true, description: "Delete entry from keys array", document: `{"a": 1, "b": 2, "c": 3}`, expression: `keys | del(.[] | select(.=="b"))`, expected: []string{ "D0, P[], (!!seq)::- \"a\"\n- \"c\"\n", }, }, { skipDoc: true, description: "Delete entry after flattening an array", document: `[1,[2],[[3]]]`, expression: `flatten | del(.[2])`, expected: []string{ "D0, P[], (!!seq)::[1, 2]\n", }, }, { skipDoc: true, document: `a: [10,x,10, 10, x, 10]`, expression: `del(.a[] | select(. == 10))`, expected: []string{ "D0, P[], (!!map)::a: [x, x]\n", }, }, { skipDoc: true, document: `a: null`, expression: `del(..)`, expected: []string{}, }, { skipDoc: true, document: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`, expression: `del(..)`, expected: []string{}, }, { skipDoc: true, document: `a: {thing1: yep, thing2: cool, thing3: hi, b: {thing1: cool, great: huh}}`, expression: `del(.. | select(tag == "!!map") | (.b.thing1,.thing2))`, expected: []string{ "D0, P[], (!!map)::a: {thing1: yep, thing3: hi, b: {great: huh}}\n", }, }, { description: "Delete nested entry in array", document: `[{a: cat, b: dog}]`, expression: `del(.[0].a)`, expected: []string{ "D0, P[], (!!seq)::[{b: dog}]\n", }, }, { description: "Delete no matches", document: `{a: cat, b: dog}`, expression: `del(.c)`, expected: []string{ "D0, P[], (!!map)::{a: cat, b: dog}\n", }, }, { description: "Delete matching entries", document: `{a: cat, b: dog, c: bat}`, expression: `del( .[] | select(. == "*at") )`, expected: []string{ "D0, P[], (!!map)::{b: dog}\n", }, }, { description: "Recursively delete matching keys", document: `{a: {name: frog, b: {name: blog, age: 12}}}`, expression: `del(.. | select(has("name")).name)`, expected: []string{ "D0, P[], (!!map)::{a: {b: {age: 12}}}\n", }, }, { skipDoc: true, description: "Repeatedly delete the first element of a list", document: `a: [0, 1, 2, 3]`, expression: `del(.a[0]) | del(.a[0])`, expected: []string{ "D0, P[], (!!map)::a: [2, 3]\n", }, }, } func TestDeleteOperatorScenarios(t *testing.T) { for _, tt := range deleteOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "delete", deleteOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_divide.go ================================================ package yqlib import ( "fmt" "strconv" "strings" ) func divideOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Divide operator") return crossFunction(d, context.ReadOnlyClone(), expressionNode, divide, false) } func divide(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if lhs.Tag == "!!null" { return nil, fmt.Errorf("%v (%v) cannot be divided by %v (%v)", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath()) } target := lhs.CopyWithoutContent() if lhs.Kind == ScalarNode && rhs.Kind == ScalarNode { if err := divideScalars(target, lhs, rhs); err != nil { return nil, err } } else { return nil, fmt.Errorf("%v (%v) cannot be divided by %v (%v)", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath()) } return target, nil } func divideScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { lhsTag := lhs.Tag rhsTag := rhs.guessTagFromCustomType() lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() lhsIsCustom = true } if lhsTag == "!!str" && rhsTag == "!!str" { tKind, tTag, res := split(lhs.Value, rhs.Value) target.Kind = tKind target.Tag = tTag target.AddChildren(res) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { target.Kind = ScalarNode target.Style = lhs.Style lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return err } quotient := lhsNum / rhsNum if lhsIsCustom { target.Tag = lhs.Tag } else { target.Tag = "!!float" } target.Value = fmt.Sprintf("%v", quotient) } else { return fmt.Errorf("%v cannot be divided by %v", lhsTag, rhsTag) } return nil } ================================================ FILE: pkg/yqlib/operator_divide_test.go ================================================ package yqlib import ( "testing" ) var divideOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `[{a: foo_bar, b: _}, {a: 4, b: 2}]`, expression: ".[] | .a / .b", expected: []string{ "D0, P[0 a], (!!seq)::- foo\n- bar\n", "D0, P[1 a], (!!float)::2\n", }, }, { skipDoc: true, document: `{}`, expression: "(.a / .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { description: "String split", document: `{a: cat_meow, b: _}`, expression: `.c = .a / .b`, expected: []string{ "D0, P[], (!!map)::{a: cat_meow, b: _, c: [cat, meow]}\n", }, }, { description: "Number division", subdescription: "The result during division is calculated as a float", document: `{a: 12, b: 2.5}`, expression: `.a = .a / .b`, expected: []string{ "D0, P[], (!!map)::{a: 4.8, b: 2.5}\n", }, }, { description: "Number division by zero", subdescription: "Dividing by zero results in +Inf or -Inf", document: `{a: 1, b: -1}`, expression: `.a = .a / 0 | .b = .b / 0`, expected: []string{ "D0, P[], (!!map)::{a: !!float +Inf, b: !!float -Inf}\n", }, }, { skipDoc: true, description: "Custom types: that are really strings", document: "a: !horse cat_meow\nb: !goat _", expression: `.a = .a / .b`, expected: []string{ "D0, P[], (!!map)::a: !horse\n - cat\n - meow\nb: !goat _\n", }, }, { skipDoc: true, description: "Custom types: that are really numbers", document: "a: !horse 1.2\nb: !goat 2.3", expression: `.a = .a / .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 0.5217391304347826\nb: !goat 2.3\n", }, }, { skipDoc: true, document: "a: 2\nb: !goat 2.3", expression: `.a = .a / .b`, expected: []string{ "D0, P[], (!!map)::a: 0.8695652173913044\nb: !goat 2.3\n", }, }, { skipDoc: true, description: "Custom types: that are really ints", document: "a: !horse 2\nb: !goat 3", expression: `.a = .a / .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 0.6666666666666666\nb: !goat 3\n", }, }, { skipDoc: true, description: "Keep anchors", document: "a: &horse [1]", expression: `.a[1] = .a[0] / 2`, expected: []string{ "D0, P[], (!!map)::a: &horse [1, 0.5]\n", }, }, { skipDoc: true, description: "Divide int by string", document: "a: 123\nb: '2'", expression: `.a / .b`, expectedError: "!!int cannot be divided by !!str", }, { skipDoc: true, description: "Divide string by int", document: "a: 2\nb: '123'", expression: `.b / .a`, expectedError: "!!str cannot be divided by !!int", }, { skipDoc: true, description: "Divide map by int", document: "a: {\"a\":1}\nb: 2", expression: `.a / .b`, expectedError: "!!map (a) cannot be divided by !!int (b)", }, { skipDoc: true, description: "Divide array by str", document: "a: [1,2]\nb: '2'", expression: `.a / .b`, expectedError: "!!seq (a) cannot be divided by !!str (b)", }, } func TestDivideOperatorScenarios(t *testing.T) { for _, tt := range divideOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "divide", divideOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_document_index.go ================================================ package yqlib import ( "container/list" "fmt" ) func getDocumentIndexOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) scalar := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.GetDocument())) results.PushBack(scalar) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_document_index_test.go ================================================ package yqlib import ( "testing" ) var documentIndexScenarios = []expressionScenario{ { description: "Retrieve a document index", document: "a: cat\n---\na: frog\n", expression: `.a | document_index`, expected: []string{ "D0, P[a], (!!int)::0\n", "D1, P[a], (!!int)::1\n", }, }, { description: "Retrieve a document index, shorthand", document: "a: cat\n---\na: frog\n", expression: `.a | di`, expected: []string{ "D0, P[a], (!!int)::0\n", "D1, P[a], (!!int)::1\n", }, }, { description: "Filter by document index", document: "a: cat\n---\na: frog\n", expression: `select(document_index == 1)`, expected: []string{ "D1, P[], (!!map)::a: frog\n", }, }, { description: "Filter by document index shorthand", document: "a: cat\n---\na: frog\n", expression: `select(di == 1)`, expected: []string{ "D1, P[], (!!map)::a: frog\n", }, }, { description: "Print Document Index with matches", document: "a: cat\n---\na: frog\n", expression: `.a | ({"match": ., "doc": document_index})`, expected: []string{ "D0, P[], (!!map)::match: cat\ndoc: 0\n", "D1, P[], (!!map)::match: frog\ndoc: 1\n", }, }, } func TestDocumentIndexScenarios(t *testing.T) { for _, tt := range documentIndexScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "document-index", documentIndexScenarios) } ================================================ FILE: pkg/yqlib/operator_encoder_decoder.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "errors" "regexp" "strings" ) func configureEncoder(format *Format, indent int) Encoder { switch format { case JSONFormat: prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = indent prefs.ColorsEnabled = false prefs.UnwrapScalar = false return NewJSONEncoder(prefs) case YamlFormat: var prefs = ConfiguredYamlPreferences.Copy() prefs.Indent = indent prefs.ColorsEnabled = false return NewYamlEncoder(prefs) case XMLFormat: var xmlPrefs = ConfiguredXMLPreferences.Copy() xmlPrefs.Indent = indent return NewXMLEncoder(xmlPrefs) } return format.EncoderFactory() } func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { var output bytes.Buffer log.Debug("printing with indent: %v", prefs.indent) encoder := configureEncoder(prefs.format, prefs.indent) if encoder == nil { return "", errors.New("no support for output format") } printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output))) err := printer.PrintResults(candidate.AsList()) return output.String(), err } type encoderPreferences struct { format *Format indent int } /* encodes object as yaml string */ var chomper = regexp.MustCompile("\n+$") func encodeOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { preferences := expressionNode.Operation.Preferences.(encoderPreferences) var results = list.New() hasOnlyOneNewLine := regexp.MustCompile("[^\n].*\n$") endWithNewLine := regexp.MustCompile(".*\n$") for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) stringValue, err := encodeToString(candidate, preferences) if err != nil { return Context{}, err } // remove trailing newlines if needed. // check if we originally decoded this path, and the original thing had a single line. originalList := context.GetVariable("decoded: " + candidate.GetKey()) if originalList != nil && originalList.Len() > 0 && hasOnlyOneNewLine.MatchString(stringValue) { original := originalList.Front().Value.(*CandidateNode) // original block did not have a newline at the end, get rid of this one too if !endWithNewLine.MatchString(original.Value) { stringValue = chomper.ReplaceAllString(stringValue, "") } } // dont print a newline when printing json on a single line. if (preferences.format == JSONFormat && preferences.indent == 0) || preferences.format == CSVFormat || preferences.format == TSVFormat { stringValue = chomper.ReplaceAllString(stringValue, "") } results.PushBack(candidate.CreateReplacement(ScalarNode, "!!str", stringValue)) } return context.ChildContext(results), nil } type decoderPreferences struct { format *Format } /* takes a string and decodes it back into an object */ func decodeOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { preferences := expressionNode.Operation.Preferences.(decoderPreferences) decoder := preferences.format.DecoderFactory() if decoder == nil { return Context{}, errors.New("no support for input format") } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) context.SetVariable("decoded: "+candidate.GetKey(), candidate.AsList()) log.Debugf("got: [%v]", candidate.Value) err := decoder.Init(strings.NewReader(candidate.Value)) if err != nil { return Context{}, err } node, errorReading := decoder.Decode() if errorReading != nil { return Context{}, errorReading } node.Key = candidate.Key node.Parent = candidate.Parent results.PushBack(node) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_encoder_decoder_test.go ================================================ package yqlib import ( "testing" ) var prefix = "D0, P[], (!!map)::a:\n cool:\n bob: dylan\n" var encoderDecoderOperatorScenarios = []expressionScenario{ { requiresFormat: "json", description: "Encode value as json string", document: `{a: {cool: "thing"}}`, expression: `.b = (.a | to_json)`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: "{\n \"cool\": \"thing\"\n}\n"} `, }, }, { requiresFormat: "json", description: "Encode value as json string, on one line", subdescription: "Pass in a 0 indent to print json on a single line.", document: `{a: {cool: "thing"}}`, expression: `.b = (.a | to_json(0))`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} `, }, }, { requiresFormat: "json", description: "Encode value as json string, on one line shorthand", subdescription: "Pass in a 0 indent to print json on a single line.", document: `{a: {cool: "thing"}}`, expression: `.b = (.a | @json)`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: '{"cool":"thing"}'} `, }, }, { requiresFormat: "json", description: "Decode a json encoded string", subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.", document: `a: '{"cool":"thing"}'`, expression: `.a | from_json | ... style=""`, expected: []string{ "D0, P[a], (!!map)::cool: thing\n", }, }, { skipDoc: true, document: `{a: {cool: "thing"}}`, expression: `.b = (.a | to_props)`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: "cool = thing\n"} `, }, }, { description: "Encode value as props string", document: `{a: {cool: "thing"}}`, expression: `.b = (.a | @props)`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: "cool = thing\n"} `, }, }, { description: "Decode props encoded string", document: `a: "cats=great\ndogs=cool as well"`, expression: `.a |= @propsd`, expected: []string{ "D0, P[], (!!map)::a:\n cats: great\n dogs: cool as well\n", }, }, { description: "Decode csv encoded string", document: `a: "cats,dogs\ngreat,cool as well"`, expression: `.a |= @csvd`, expected: []string{ "D0, P[], (!!map)::a:\n - cats: great\n dogs: cool as well\n", }, }, { description: "Decode tsv encoded string", document: `a: "cats dogs\ngreat cool as well"`, expression: `.a |= @tsvd`, expected: []string{ "D0, P[], (!!map)::a:\n - cats: great\n dogs: cool as well\n", }, }, { skipDoc: true, document: "a:\n cool:\n bob: dylan", expression: `.b = (.a | @yaml)`, expected: []string{ prefix + "b: |\n cool:\n bob: dylan\n", }, }, { description: "Encode value as yaml string", subdescription: "Indent defaults to 2", document: "a:\n cool:\n bob: dylan", expression: `.b = (.a | to_yaml)`, expected: []string{ prefix + "b: |\n cool:\n bob: dylan\n", }, }, { description: "Encode value as yaml string, with custom indentation", subdescription: "You can specify the indentation level as the first parameter.", document: "a:\n cool:\n bob: dylan", expression: `.b = (.a | to_yaml(8))`, expected: []string{ prefix + "b: |\n cool:\n bob: dylan\n", }, }, { skipDoc: true, document: `{a: {cool: "thing"}}`, expression: `.b = (.a | to_yaml)`, expected: []string{ `D0, P[], (!!map)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"} `, }, }, { description: "Decode a yaml encoded string", document: `a: "foo: bar"`, expression: `.b = (.a | from_yaml)`, expected: []string{ "D0, P[], (!!map)::a: \"foo: bar\"\nb:\n foo: bar\n", }, }, { description: "Update a multiline encoded yaml string", dontFormatInputForDoc: true, document: "a: |\n foo: bar\n baz: dog\n", expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, expected: []string{ "D0, P[], (!!map)::a: |\n foo: cat\n baz: dog\n", }, }, { skipDoc: true, dontFormatInputForDoc: true, document: "a: |-\n foo: bar\n baz: dog\n", expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, expected: []string{ "D0, P[], (!!map)::a: |-\n foo: cat\n baz: dog\n", }, }, { description: "Update a single line encoded yaml string", dontFormatInputForDoc: true, document: "a: 'foo: bar'", expression: `.a |= (from_yaml | .foo = "cat" | to_yaml)`, expected: []string{ "D0, P[], (!!map)::a: 'foo: cat'\n", }, }, { description: "Encode array of scalars as csv string", subdescription: "Scalars are strings, numbers and booleans.", document: `[cat, "thing1,thing2", true, 3.40]`, expression: `@csv`, expected: []string{ "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n", }, }, { description: "Encode array of arrays as csv string", document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, expression: `@csv`, expected: []string{ "D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n", }, }, { description: "Encode array of arrays as tsv string", subdescription: "Scalars are strings, numbers and booleans.", document: `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`, expression: `@tsv`, expected: []string{ "D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n", }, }, { skipDoc: true, dontFormatInputForDoc: true, document: "a: \"foo: bar\"", expression: `.a |= (from_yaml | .foo = {"a": "frog"} | to_yaml)`, expected: []string{ "D0, P[], (!!map)::a: \"foo:\\n a: frog\"\n", }, }, { requiresFormat: "xml", description: "Encode value as xml string", document: `{a: {cool: {foo: "bar", +@id: hi}}}`, expression: `.a | to_xml`, expected: []string{ "D0, P[a], (!!str)::\n bar\n\n\n", }, }, { requiresFormat: "xml", description: "Encode value as xml string on a single line", document: `{a: {cool: {foo: "bar", +@id: hi}}}`, expression: `.a | @xml`, expected: []string{ "D0, P[a], (!!str)::bar\n\n", }, }, { requiresFormat: "xml", description: "Encode value as xml string with custom indentation", document: `{a: {cool: {foo: "bar", +@id: hi}}}`, expression: `{"cat": .a | to_xml(1)}`, expected: []string{ "D0, P[], (!!map)::cat: |\n \n bar\n \n", }, }, { requiresFormat: "xml", description: "Decode a xml encoded string", document: `a: "bar"`, expression: `.b = (.a | from_xml)`, expected: []string{ "D0, P[], (!!map)::a: \"bar\"\nb:\n foo: bar\n", }, }, { description: "Encode a string to base64", document: "coolData: a special string", expression: ".coolData | @base64", expected: []string{ "D0, P[coolData], (!!str)::YSBzcGVjaWFsIHN0cmluZw==\n", }, }, { description: "Encode a yaml document to base64", subdescription: "Pipe through @yaml first to convert to a string, then use @base64 to encode it.", document: "a: apple", expression: "@yaml | @base64", expected: []string{ "D0, P[], (!!str)::YTogYXBwbGUK\n", }, }, { description: "Encode a string to uri", document: "coolData: this has & special () characters *", expression: ".coolData | @uri", expected: []string{ "D0, P[coolData], (!!str)::this+has+%26+special+%28%29+characters+%2A\n", }, }, { description: "Decode a URI to a string", document: "this+has+%26+special+%28%29+characters+%2A", expression: "@urid", expected: []string{ "D0, P[], (!!str)::this has & special () characters *\n", }, }, { description: "Encode a string to sh", subdescription: "Sh/Bash friendly string", document: "coolData: strings with spaces and a 'quote'", expression: ".coolData | @sh", expected: []string{ "D0, P[coolData], (!!str)::strings' with spaces and a '\\'quote\\'\n", }, }, { description: "Encode a string to sh", subdescription: "Watch out for stray '' (empty strings)", document: "coolData: \"'starts, contains more '' and ends with a quote'\"", expression: ".coolData | @sh", expected: []string{ "D0, P[coolData], (!!str)::\\'starts,' contains more '\\'\\'' and ends with a quote'\\'\n", }, skipDoc: true, }, { description: "Decode a base64 encoded string", subdescription: "Decoded data is assumed to be a string.", document: "coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig==", expression: ".coolData | @base64d", expected: []string{ "D0, P[coolData], (!!str)::Works with UTF-16 😊\n", }, }, { description: "Decode a base64 encoded yaml document", subdescription: "Pipe through `from_yaml` to parse the decoded base64 string as a yaml document.", document: "coolData: YTogYXBwbGUK", expression: ".coolData |= (@base64d | from_yaml)", expected: []string{ "D0, P[], (!!map)::coolData:\n a: apple\n", }, }, { description: "empty base64 decode", skipDoc: true, expression: `"" | @base64d`, expected: []string{ "D0, P[], (!!str)::\n", }, }, { description: "base64 missing padding test", skipDoc: true, expression: `"Y2F0cw" | @base64d`, expected: []string{ "D0, P[], (!!str)::cats\n", }, }, { description: "base64 missing padding test", skipDoc: true, expression: `"cats" | @base64 | @base64d`, expected: []string{ "D0, P[], (!!str)::cats\n", }, }, { requiresFormat: "xml", description: "empty xml decode", skipDoc: true, expression: `"" | @xmld`, expected: []string{ "D0, P[], (!!null)::\n", }, }, } func TestEncoderDecoderOperatorScenarios(t *testing.T) { for _, tt := range encoderDecoderOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "encode-decode", encoderDecoderOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_entries.go ================================================ package yqlib import ( "container/list" "fmt" ) func entrySeqFor(key *CandidateNode, value *CandidateNode) *CandidateNode { var keyKey = &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "key"} var valueKey = &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "value"} candidate := &CandidateNode{Kind: MappingNode, Tag: "!!map"} candidate.AddKeyValueChild(keyKey, key) candidate.AddKeyValueChild(valueKey, value) return candidate } func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0) var contents = candidateNode.Content for index := 0; index < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] sequence.AddChild(entrySeqFor(key, value)) } return sequence } func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode { var sequence = candidateNode.CreateReplacementWithComments(SequenceNode, "!!seq", 0) var contents = candidateNode.Content for index := 0; index < len(contents); index = index + 1 { key := &CandidateNode{Kind: ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)} value := contents[index] sequence.AddChild(entrySeqFor(key, value)) } return sequence } func toEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) switch candidate.Kind { case MappingNode: results.PushBack(toEntriesFromMap(candidate)) case SequenceNode: results.PushBack(toEntriesfromSeq(candidate)) default: if candidate.Tag != "!!null" { return Context{}, fmt.Errorf("%v has no keys", candidate.Tag) } } } return context.ChildContext(results), nil } func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *CandidateNode, error) { prefs := traversePreferences{DontAutoCreate: true} keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false) if err != nil { return nil, nil, err } else if keyResults.Len() != 1 { return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position) } valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false) if err != nil { return nil, nil, err } else if valueResults.Len() != 1 { return nil, nil, fmt.Errorf("expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position) } return keyResults.Front().Value.(*CandidateNode), valueResults.Front().Value.(*CandidateNode), nil } func fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) { var node = candidateNode.CopyWithoutContent() var contents = candidateNode.Content for index := 0; index < len(contents); index = index + 1 { key, value, err := parseEntry(contents[index], index) if err != nil { return nil, err } node.AddKeyValueChild(key, value) } node.Kind = MappingNode node.Tag = "!!map" return node, nil } func fromEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) switch candidate.Kind { case SequenceNode: mapResult, err := fromEntries(candidate) if err != nil { return Context{}, err } results.PushBack(mapResult) default: return Context{}, fmt.Errorf("from entries only runs against arrays") } } return context.ChildContext(results), nil } func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { //to_entries on the context toEntries, err := toEntriesOperator(d, context, expressionNode) if err != nil { return Context{}, err } var results = list.New() for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) //run expression against entries // splat toEntries and pipe it into Rhs splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) if err != nil { return Context{}, err } newResults := list.New() for itemEl := splatted.MatchingNodes.Front(); itemEl != nil; itemEl = itemEl.Next() { result, err := d.GetMatchingNodes(splatted.SingleChildContext(itemEl.Value.(*CandidateNode)), expressionNode.RHS) if err != nil { return Context{}, err } newResults.PushBackList(result.MatchingNodes) } selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} collected, err := collectTogether(d, splatted.ChildContext(newResults), selfExpression) if err != nil { return Context{}, err } log.Debug("candidate %v", NodeToString(candidate)) log.Debug("candidate leading content: %v", candidate.LeadingContent) collected.LeadingContent = candidate.LeadingContent log.Debug("candidate FootComment: [%v]", candidate.FootComment) collected.HeadComment = candidate.HeadComment collected.FootComment = candidate.FootComment log.Debugf("collected %v", collected.LeadingContent) fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode) if err != nil { return Context{}, err } results.PushBackList(fromEntries.MatchingNodes) } //from_entries on the result return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_entries_test.go ================================================ package yqlib import ( "testing" ) var entriesOperatorScenarios = []expressionScenario{ { description: "to_entries splat", skipDoc: true, document: `{a: 1, b: 2}`, expression: `to_entries[]`, expected: []string{ "D0, P[0], (!!map)::key: a\nvalue: 1\n", "D0, P[1], (!!map)::key: b\nvalue: 2\n", }, }, { description: "to_entries, delete key", skipDoc: true, document: `{a: 1, b: 2}`, expression: `to_entries | map(del(.key))`, expected: []string{ "D0, P[], (!!seq)::- value: 1\n- value: 2\n", }, }, { description: "to_entries Map", document: `{a: 1, b: 2}`, expression: `to_entries`, expected: []string{ "D0, P[], (!!seq)::- key: a\n value: 1\n- key: b\n value: 2\n", }, }, { description: "to_entries Array", document: `[a, b]`, expression: `to_entries`, expected: []string{ "D0, P[], (!!seq)::- key: 0\n value: a\n- key: 1\n value: b\n", }, }, { description: "to_entries null", document: `null`, expression: `to_entries`, expected: []string{}, }, { description: "from_entries map", document: `{a: 1, b: 2}`, expression: `to_entries | from_entries`, expected: []string{ "D0, P[], (!!map)::a: 1\nb: 2\n", }, }, { description: "from_entries with numeric key indices", subdescription: "from_entries always creates a map, even for numeric keys", document: `[a,b]`, expression: `to_entries | from_entries`, expected: []string{ "D0, P[], (!!map)::0: a\n1: b\n", }, }, { description: "Use with_entries to update keys", document: `{a: 1, b: 2}`, // expression: `to_entries | with(.[]; .key |= "KEY_" + .) | from_entries`, expression: `with_entries(.key |= "KEY_" + .)`, expected: []string{ "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", }, }, { description: "Use with_entries to update keys recursively", document: `{a: 1, b: {b_a: nested, b_b: thing}}`, expression: `(.. | select(tag=="!!map")) |= with_entries(.key |= "KEY_" + .)`, subdescription: "We use (.. | select(tag=\"map\")) to find all the maps in the doc, then |= to update each one of those maps. In the update, with_entries is used.", expected: []string{ "D0, P[], (!!map)::{KEY_a: 1, KEY_b: {KEY_b_a: nested, KEY_b_b: thing}}\n", }, }, { skipDoc: true, description: "Use with_entries to update keys comment", document: `{a: 1, b: 2}`, expression: `with_entries(.key headComment= .value)`, expected: []string{ "D0, P[], (!!map)::# 1\na: 1\n# 2\nb: 2\n", }, }, { description: "Custom sort map keys", subdescription: "Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.", document: `{a: 1, c: 3, b: 2}`, expression: `to_entries | sort_by(.key) | reverse | from_entries`, expected: []string{ "D0, P[], (!!map)::c: 3\nb: 2\na: 1\n", }, }, { skipDoc: true, document: `{a: 1, b: 2}`, document2: `{c: 1, d: 2}`, expression: `with_entries(.key |= "KEY_" + .)`, expected: []string{ "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", "D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n", }, }, { skipDoc: true, document: `[{a: 1, b: 2}, {c: 1, d: 2}]`, expression: `.[] | with_entries(.key |= "KEY_" + .)`, expected: []string{ "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", "D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n", }, }, { description: "Use with_entries to filter the map", document: `{a: { b: bird }, c: { d: dog }}`, expression: `with_entries(select(.value | has("b")))`, expected: []string{ "D0, P[], (!!map)::a: {b: bird}\n", }, }, { description: "Use with_entries to filter the map; head comment", skipDoc: true, document: "# abc\n{a: { b: bird }, c: { d: dog }}\n# xyz", expression: `with_entries(select(.value | has("b")))`, expected: []string{ "D0, P[], (!!map)::# abc\na: {b: bird}\n# xyz\n", }, }, } func TestEntriesOperatorScenarios(t *testing.T) { for _, tt := range entriesOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "entries", entriesOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_env.go ================================================ package yqlib import ( "container/list" "fmt" "os" "strings" parse "github.com/a8m/envsubst/parse" ) type envOpPreferences struct { StringValue bool NoUnset bool NoEmpty bool FailFast bool } func envOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if ConfiguredSecurityPreferences.DisableEnvOps { return Context{}, fmt.Errorf("env operations have been disabled") } envName := expressionNode.Operation.CandidateNode.Value log.Debug("EnvOperator, env name:", envName) rawValue := os.Getenv(envName) preferences := expressionNode.Operation.Preferences.(envOpPreferences) var node *CandidateNode if preferences.StringValue { node = &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: rawValue, } } else if rawValue == "" { return Context{}, fmt.Errorf("value for env variable '%v' not provided in env()", envName) } else { decoder := NewYamlDecoder(ConfiguredYamlPreferences) if err := decoder.Init(strings.NewReader(rawValue)); err != nil { return Context{}, err } var err error node, err = decoder.Decode() if err != nil { return Context{}, err } } log.Debug("ENV tag", node.Tag) log.Debug("ENV value", node.Value) log.Debug("ENV Kind", node.Kind) return context.SingleChildContext(node), nil } func envsubstOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if ConfiguredSecurityPreferences.DisableEnvOps { return Context{}, fmt.Errorf("env operations have been disabled") } var results = list.New() preferences := envOpPreferences{} if expressionNode.Operation.Preferences != nil { preferences = expressionNode.Operation.Preferences.(envOpPreferences) } parser := parse.New("string", os.Environ(), &parse.Restrictions{NoUnset: preferences.NoUnset, NoEmpty: preferences.NoEmpty}) if preferences.FailFast { parser.Mode = parse.Quick } else { parser.Mode = parse.AllErrors } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.Tag != "!!str" { log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value) return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } value, err := parser.Parse(node.Value) if err != nil { return Context{}, err } result := node.CreateReplacement(ScalarNode, "!!str", value) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_env_test.go ================================================ package yqlib import ( "testing" ) var envOperatorScenarios = []expressionScenario{ { description: "Read string environment variable", skipDoc: true, environmentVariables: map[string]string{"myenv": "[cat,dog]"}, expression: `env(myenv)[]`, expected: []string{ "D0, P[0], (!!str)::cat\n", "D0, P[1], (!!str)::dog\n", }, }, { description: "Read string environment variable", environmentVariables: map[string]string{"myenv": "cat meow"}, expression: `.a = env(myenv)`, expected: []string{ "D0, P[], ()::a: cat meow\n", }, }, { description: "Read boolean environment variable", environmentVariables: map[string]string{"myenv": "true"}, expression: `.a = env(myenv)`, expected: []string{ "D0, P[], ()::a: true\n", }, }, { description: "Read numeric environment variable", environmentVariables: map[string]string{"myenv": "12"}, expression: `.a = env(myenv)`, expected: []string{ "D0, P[], ()::a: 12\n", }, }, { description: "Read yaml environment variable", environmentVariables: map[string]string{"myenv": "{b: fish}"}, expression: `.a = env(myenv)`, expected: []string{ "D0, P[], ()::a: {b: fish}\n", }, }, { description: "Read boolean environment variable as a string", environmentVariables: map[string]string{"myenv": "true"}, expression: `.a = strenv(myenv)`, expected: []string{ "D0, P[], ()::a: \"true\"\n", }, }, { description: "Read numeric environment variable as a string", environmentVariables: map[string]string{"myenv": "12"}, expression: `.a = strenv(myenv)`, expected: []string{ "D0, P[], ()::a: \"12\"\n", }, }, { description: "Dynamically update a path from an environment variable", subdescription: "The env variable can be any valid yq expression.", document: `{a: {b: [{name: dog}, {name: cat}]}}`, environmentVariables: map[string]string{"pathEnv": ".a.b[0].name", "valueEnv": "moo"}, expression: `eval(strenv(pathEnv)) = strenv(valueEnv)`, expected: []string{ "D0, P[], (!!map)::{a: {b: [{name: moo}, {name: cat}]}}\n", }, }, { description: "Dynamic key lookup with environment variable", environmentVariables: map[string]string{"myenv": "cat"}, document: `{cat: meow, dog: woof}`, expression: `.[env(myenv)]`, expected: []string{ "D0, P[cat], (!!str)::meow\n", }, }, { description: "Replace strings with envsubst", environmentVariables: map[string]string{"myenv": "cat"}, expression: `"the ${myenv} meows" | envsubst`, expected: []string{ "D0, P[], (!!str)::the cat meows\n", }, }, { description: "Replace strings with envsubst, missing variables", expression: `"the ${myenvnonexisting} meows" | envsubst`, expected: []string{ "D0, P[], (!!str)::the meows\n", }, }, { description: "Replace strings with envsubst(nu), missing variables", subdescription: "(nu) not unset, will fail if there are unset (missing) variables", expression: `"the ${myenvnonexisting} meows" | envsubst(nu)`, expectedError: "variable ${myenvnonexisting} not set", }, { description: "Replace strings with envsubst(ne), missing variables", subdescription: "(ne) not empty, only validates set variables", expression: `"the ${myenvnonexisting} meows" | envsubst(ne)`, expected: []string{ "D0, P[], (!!str)::the meows\n", }, }, { description: "Replace strings with envsubst(ne), empty variable", subdescription: "(ne) not empty, will fail if a references variable is empty", environmentVariables: map[string]string{"myenv": ""}, expression: `"the ${myenv} meows" | envsubst(ne)`, expectedError: "variable ${myenv} set but empty", }, { description: "Replace strings with envsubst, missing variables with defaults", expression: `"the ${myenvnonexisting-dog} meows" | envsubst`, expected: []string{ "D0, P[], (!!str)::the dog meows\n", }, }, { description: "Replace strings with envsubst(nu), missing variables with defaults", subdescription: "Having a default specified skips over the missing variable.", expression: `"the ${myenvnonexisting-dog} meows" | envsubst(nu)`, expected: []string{ "D0, P[], (!!str)::the dog meows\n", }, }, { description: "Replace strings with envsubst(ne), missing variables with defaults", subdescription: "Fails, because the variable is explicitly set to blank.", environmentVariables: map[string]string{"myEmptyEnv": ""}, expression: `"the ${myEmptyEnv-dog} meows" | envsubst(ne)`, expectedError: "variable ${myEmptyEnv} set but empty", }, { description: "Replace string environment variable in document", environmentVariables: map[string]string{"myenv": "cat meow"}, document: "{v: \"${myenv}\"}", expression: `.v |= envsubst`, expected: []string{ "D0, P[], (!!map)::{v: \"cat meow\"}\n", }, }, { description: "(Default) Return all envsubst errors", subdescription: "By default, all errors are returned at once.", expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu)`, expectedError: "variable ${notThere} not set\nvariable ${alsoNotThere} not set", }, { description: "Fail fast, return the first envsubst error (and abort)", expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`, expectedError: "variable ${notThere} not set", }, { description: "with header/footer", skipDoc: true, environmentVariables: map[string]string{"myenv": "cat meow"}, document: "# abc\n{v: \"${myenv}\"}\n# xyz\n", expression: `(.. | select(tag == "!!str")) |= envsubst`, expected: []string{ "D0, P[], (!!map)::# abc\n{v: \"cat meow\"}\n# xyz\n", }, }, } func TestEnvOperatorScenarios(t *testing.T) { for _, tt := range envOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "env-variable-operators", envOperatorScenarios) } var envOperatorSecurityDisabledScenarios = []expressionScenario{ { description: "env() operation fails when security is enabled", subdescription: "Use `--security-disable-env-ops` to disable env operations for security.", expression: `env("MYENV")`, expectedError: "env operations have been disabled", }, { description: "strenv() operation fails when security is enabled", subdescription: "Use `--security-disable-env-ops` to disable env operations for security.", expression: `strenv("MYENV")`, expectedError: "env operations have been disabled", }, { description: "envsubst() operation fails when security is enabled", subdescription: "Use `--security-disable-env-ops` to disable env operations for security.", expression: `"value: ${MYENV}" | envsubst`, expectedError: "env operations have been disabled", }, } func TestEnvOperatorSecurityDisabledScenarios(t *testing.T) { // Save original security preferences originalDisableEnvOps := ConfiguredSecurityPreferences.DisableEnvOps defer func() { ConfiguredSecurityPreferences.DisableEnvOps = originalDisableEnvOps }() // Test that env() fails when DisableEnvOps is true ConfiguredSecurityPreferences.DisableEnvOps = true for _, tt := range envOperatorSecurityDisabledScenarios { testScenario(t, &tt) } appendOperatorDocumentScenario(t, "env-variable-operators", envOperatorSecurityDisabledScenarios) } ================================================ FILE: pkg/yqlib/operator_equals.go ================================================ package yqlib func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("equalsOperation") return crossFunction(d, context, expressionNode, isEquals(false), true) } func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { return func(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { value := false log.Debugf("isEquals cross function") if lhs == nil && rhs == nil { log.Debugf("both are nil") owner := &CandidateNode{} return createBooleanCandidate(owner, !flip), nil } else if lhs == nil { log.Debugf("lhs nil, but rhs is not") value := rhs.Tag == "!!null" if flip { value = !value } return createBooleanCandidate(rhs, value), nil } else if rhs == nil { log.Debugf("lhs not nil, but rhs is") value := lhs.Tag == "!!null" if flip { value = !value } return createBooleanCandidate(lhs, value), nil } if lhs.Tag == "!!null" { value = (rhs.Tag == "!!null") } else if lhs.Kind == ScalarNode && rhs.Kind == ScalarNode { value = matchKey(lhs.Value, rhs.Value) } log.Debugf("%v == %v ? %v", NodeToString(lhs), NodeToString(rhs), value) if flip { value = !value } return createBooleanCandidate(lhs, value), nil } } func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("notEqualsOperator") return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true) } ================================================ FILE: pkg/yqlib/operator_equals_test.go ================================================ package yqlib import ( "testing" ) var equalsOperatorScenarios = []expressionScenario{ { skipDoc: true, expression: ".a == .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { expression: `(.k | length) == 0`, skipDoc: true, expected: []string{ "D0, P[k], (!!bool)::true\n", }, }, { skipDoc: true, document: `a: cat`, expression: ".a == .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: `a: cat`, expression: ".b == .a", expected: []string{ "D0, P[b], (!!bool)::false\n", }, }, { skipDoc: true, document: "cat", document2: "dog", expression: "select(fi==0) == select(fi==1)", expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: "{}", expression: "(.a == .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: "{}", expression: ".a == .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "{}", expression: "(.a != .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: "{}", expression: ".a != .b", expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { skipDoc: true, document: "{a: {b: 10}}", expression: "select(.c != null)", expected: []string{}, }, { skipDoc: true, document: "{a: {b: 10}}", expression: "select(.d == .c)", expected: []string{ "D0, P[], (!!map)::{a: {b: 10}}\n", }, }, { skipDoc: true, document: "{a: {b: 10}}", expression: "select(null == .c)", expected: []string{ "D0, P[], (!!map)::{a: {b: 10}}\n", }, }, { skipDoc: true, document: "{a: { b: {things: \"\"}, f: [1], g: [] }}", expression: ".. | select(. == \"\")", expected: []string{ "D0, P[a b things], (!!str)::\n", }, }, { description: "Match string", document: `[cat,goat,dog]`, expression: `.[] | (. == "*at")`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::true\n", "D0, P[2], (!!bool)::false\n", }, }, { description: "Don't match string", document: `[cat,goat,dog]`, expression: `.[] | (. != "*at")`, expected: []string{ "D0, P[0], (!!bool)::false\n", "D0, P[1], (!!bool)::false\n", "D0, P[2], (!!bool)::true\n", }, }, { description: "Match number", document: `[3, 4, 5]`, expression: `.[] | (. == 4)`, expected: []string{ "D0, P[0], (!!bool)::false\n", "D0, P[1], (!!bool)::true\n", "D0, P[2], (!!bool)::false\n", }, }, { description: "Don't match number", document: `[3, 4, 5]`, expression: `.[] | (. != 4)`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::false\n", "D0, P[2], (!!bool)::true\n", }, }, { skipDoc: true, document: `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`, expression: `.a | (.[].b == "apple")`, expected: []string{ "D0, P[a cat b], (!!bool)::true\n", "D0, P[a pat b], (!!bool)::false\n", }, }, { skipDoc: true, document: ``, expression: `null == null`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "Match nulls", document: ``, expression: `null == ~`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { description: "Non existent key doesn't equal a value", document: "a: frog", expression: `select(.b != "thing")`, expected: []string{ "D0, P[], (!!map)::a: frog\n", }, }, { description: "Two non existent keys are equal", document: "a: frog", expression: `select(.b == .c)`, expected: []string{ "D0, P[], (!!map)::a: frog\n", }, }, } func TestEqualOperatorScenarios(t *testing.T) { for _, tt := range equalsOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "equals", equalsOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_error.go ================================================ package yqlib import ( "errors" ) func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("errorOperation") rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } errorMessage := "aborted" if rhs.MatchingNodes.Len() > 0 { errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } return Context{}, errors.New(errorMessage) } ================================================ FILE: pkg/yqlib/operator_error_test.go ================================================ package yqlib import "testing" const validationExpression = ` with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | .numPets = env(numberOfCats) ` var errorOperatorScenarios = []expressionScenario{ { description: "Validate a particular value", document: `a: hello`, expression: `select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")`, expectedError: ".a [hello] is not howdy!", }, { description: "Validate the environment variable is a number - invalid", environmentVariables: map[string]string{"numberOfCats": "please"}, expression: `env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")`, expectedError: "numberOfCats is not a number :(", }, { description: "Validate the environment variable is a number - valid", subdescription: "`with` can be a convenient way of encapsulating validation.", environmentVariables: map[string]string{"numberOfCats": "3"}, document: "name: Bob\nfavouriteAnimal: cat\n", expression: validationExpression, expected: []string{ "D0, P[], (!!map)::name: Bob\nfavouriteAnimal: cat\nnumPets: 3\n", }, }, } func TestErrorOperatorScenarios(t *testing.T) { for _, tt := range errorOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "error", errorOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_eval.go ================================================ package yqlib import ( "container/list" ) func evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Eval") pathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } expressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len()) expIndex := 0 //parse every expression for pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() { expressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode) expressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Value) if err != nil { return Context{}, err } expIndex++ } results := list.New() for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { for expIndex = 0; expIndex < len(expressions); expIndex++ { result, err := d.GetMatchingNodes(context, expressions[expIndex]) if err != nil { return Context{}, err } results.PushBackList(result.MatchingNodes) } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_eval_test.go ================================================ package yqlib import ( "testing" ) var evalOperatorScenarios = []expressionScenario{ { description: "Dynamically evaluate a path", document: `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`, expression: `eval(.pathExp)`, expected: []string{ "D0, P[a b 1], (!!map)::{name: cat}\n", }, }, { description: "eval splat", skipDoc: true, document: `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`, expression: `eval(.pathExp)[]`, expected: []string{ "D0, P[a b 1 name], (!!str)::cat\n", }, }, { description: "Dynamically update a path from an environment variable", subdescription: "The env variable can be any valid yq expression.", document: `{a: {b: [{name: dog}, {name: cat}]}}`, environmentVariables: map[string]string{"pathEnv": ".a.b[0].name", "valueEnv": "moo"}, expression: `eval(strenv(pathEnv)) = strenv(valueEnv)`, expected: []string{ "D0, P[], (!!map)::{a: {b: [{name: moo}, {name: cat}]}}\n", }, }, } func TestEvalOperatorsScenarios(t *testing.T) { for _, tt := range evalOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "eval", evalOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_expression.go ================================================ package yqlib type expressionOpPreferences struct { expression string } func expressionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { prefs := expressionNode.Operation.Preferences.(expressionOpPreferences) expNode, err := ExpressionParser.ParseExpression(prefs.expression) if err != nil { return Context{}, err } return d.GetMatchingNodes(context, expNode) } ================================================ FILE: pkg/yqlib/operator_file.go ================================================ package yqlib import ( "container/list" "fmt" ) func getFilenameOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetFilename") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!str", candidate.GetFilename()) results.PushBack(result) } return context.ChildContext(results), nil } func getFileIndexOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetFileIndex") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.GetFileIndex())) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_file_test.go ================================================ package yqlib import ( "testing" ) var fileOperatorScenarios = []expressionScenario{ { description: "Get filename", document: `{a: cat}`, expression: `filename`, expected: []string{ "D0, P[], (!!str)::sample.yml\n", }, }, { description: "Get file index", document: `{a: cat}`, expression: `file_index`, expected: []string{ "D0, P[], (!!int)::0\n", }, }, { description: "Get file indices of multiple documents", document: `{a: cat}`, document2: `{a: cat}`, expression: `file_index`, expected: []string{ "D0, P[], (!!int)::0\n", "D0, P[], (!!int)::1\n", }, }, { description: "Get file index alias", document: `{a: cat}`, expression: `fi`, expected: []string{ "D0, P[], (!!int)::0\n", }, }, { skipDoc: true, document: "a: cat\nb: dog", expression: `.. lineComment |= filename`, expected: []string{ "D0, P[], (!!map)::a: cat # sample.yml\nb: dog # sample.yml\n", }, }, } func TestFileOperatorsScenarios(t *testing.T) { for _, tt := range fileOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "file-operators", fileOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_filter.go ================================================ package yqlib import ( "container/list" ) func filterOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("filterOperation") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) children := context.SingleChildContext(candidate) splatted, err := splat(children, traversePreferences{}) if err != nil { return Context{}, err } filtered, err := selectOperator(d, splatted, expressionNode) if err != nil { return Context{}, err } selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} collected, err := collectTogether(d, filtered, selfExpression) if err != nil { return Context{}, err } collected.Style = candidate.Style results.PushBack(collected) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_filter_test.go ================================================ package yqlib import ( "testing" ) var filterOperatorScenarios = []expressionScenario{ { description: "Filter array", document: `[1,2,3]`, expression: `filter(. < 3)`, expected: []string{ "D0, P[], (!!seq)::[1, 2]\n", }, }, { description: "Filter array splat", skipDoc: true, document: `[1,2,3]`, expression: `filter(. < 3)[]`, expected: []string{ "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", }, }, { description: "Filter map values", document: `{c: {things: cool, frog: yes}, d: {things: hot, frog: false}}`, expression: `filter(.things == "cool")`, expected: []string{ "D0, P[], (!!seq)::[{things: cool, frog: yes}]\n", }, }, { skipDoc: true, document: `[1,2,3]`, expression: `filter(. > 1)`, expected: []string{ "D0, P[], (!!seq)::[2, 3]\n", }, }, { skipDoc: true, description: "Filter array to empty", document: `[1,2,3]`, expression: `filter(. > 4)`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, description: "Filter empty array", document: `[]`, expression: `filter(. > 1)`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, } func TestFilterOperatorScenarios(t *testing.T) { for _, tt := range filterOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "filter", filterOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_first.go ================================================ package yqlib import "container/list" func firstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) // If no RHS expression is provided, simply return the first entry in candidate.Content if expressionNode == nil || expressionNode.RHS == nil { if len(candidate.Content) > 0 { results.PushBack(candidate.Content[0]) } continue } splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) if err != nil { return Context{}, err } for splatEl := splatted.MatchingNodes.Front(); splatEl != nil; splatEl = splatEl.Next() { splatCandidate := splatEl.Value.(*CandidateNode) // Create a new context for this splatted candidate splatContext := context.SingleChildContext(splatCandidate) // Evaluate the RHS expression against this splatted candidate rhs, err := d.GetMatchingNodes(splatContext, expressionNode.RHS) if err != nil { return Context{}, err } includeResult := false for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() { result := resultEl.Value.(*CandidateNode) includeResult = isTruthyNode(result) if includeResult { break } } if includeResult { results.PushBack(splatCandidate) break } } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_first_test.go ================================================ package yqlib import "testing" var firstOperatorScenarios = []expressionScenario{ { description: "First matching element from array", document: "[{a: banana},{a: cat},{a: apple}]", expression: `first(.a == "cat")`, expected: []string{ "D0, P[1], (!!map)::{a: cat}\n", }, }, { description: "First matching element from array with multiple matches", document: "[{a: banana},{a: cat, b: firstCat},{a: apple},{a: cat, b: secondCat}]", expression: `first(.a == "cat")`, expected: []string{ "D0, P[1], (!!map)::{a: cat, b: firstCat}\n", }, }, { description: "First matching element from array with numeric condition", document: "[{a: 10},{a: 100},{a: 1},{a: 101}]", expression: `first(.a > 50)`, expected: []string{ "D0, P[1], (!!map)::{a: 100}\n", }, }, { description: "First matching element from array with boolean condition", document: "[{a: false},{a: true, b: firstTrue},{a: false}, {a: true, b: secondTrue}]", expression: `first(.a == true)`, expected: []string{ "D0, P[1], (!!map)::{a: true, b: firstTrue}\n", }, }, { description: "First matching element from array with null values", document: "[{a: null},{a: cat},{a: apple}]", expression: `first(.a != null)`, expected: []string{ "D0, P[1], (!!map)::{a: cat}\n", }, }, { description: "First matching element from array with complex condition", document: "[{a: dog, b: 7},{a: cat, b: 3},{a: apple, b: 5}]", expression: `first(.b > 4 and .b < 6)`, expected: []string{ "D0, P[2], (!!map)::{a: apple, b: 5}\n", }, }, { description: "First matching element from map", document: "x: {a: banana}\ny: {a: cat}\nz: {a: apple}", expression: `first(.a == "cat")`, expected: []string{ "D0, P[y], (!!map)::{a: cat}\n", }, }, { description: "First matching element from map with numeric condition", document: "x: {a: 10}\ny: {a: 100}\nz: {a: 101}", expression: `first(.a > 50)`, expected: []string{ "D0, P[y], (!!map)::{a: 100}\n", }, }, { description: "First matching element from nested structure", document: "items: [{a: banana},{a: cat},{a: apple}]", expression: `.items | first(.a == "cat")`, expected: []string{ "D0, P[items 1], (!!map)::{a: cat}\n", }, }, { description: "First matching element with no matches", document: "[{a: banana},{a: cat},{a: apple}]", expression: `first(.a == "dog")`, expected: []string{ // No output expected when no matches }, }, { description: "First matching element from empty array", document: "[]", expression: `first(.a == "cat")`, expected: []string{ // No output expected when array is empty }, }, { description: "First matching element from scalar node", document: "hello", expression: `first(. == "hello")`, expected: []string{ // No output expected when node is scalar (no content to splat) }, }, { description: "First matching element from null node", document: "null", expression: `first(. == "hello")`, expected: []string{ // No output expected when node is null (no content to splat) }, }, { description: "First matching element with string condition", document: "[{a: banana},{a: cat},{a: apple}]", expression: `first(.a | test("^c"))`, expected: []string{ "D0, P[1], (!!map)::{a: cat}\n", }, }, { description: "First matching element with length condition", document: "[{a: hi},{a: hello},{a: world}]", expression: `first(.a | length > 4)`, expected: []string{ "D0, P[1], (!!map)::{a: hello}\n", }, }, { description: "First matching element from array of strings", document: "[banana, cat, apple]", expression: `first(. == "cat")`, expected: []string{ "D0, P[1], (!!str)::cat\n", }, }, { description: "First matching element from array of numbers", document: "[10, 100, 1]", expression: `first(. > 50)`, expected: []string{ "D0, P[1], (!!int)::100\n", }, }, { description: "First element with no filter from array", document: "[10, 100, 1]", expression: `first`, expected: []string{ "D0, P[0], (!!int)::10\n", }, }, { description: "First element with no filter from array of maps", document: "[{a: 10},{a: 100}]", expression: `first`, expected: []string{ "D0, P[0], (!!map)::{a: 10}\n", }, }, { description: "No filter on empty array returns nothing", skipDoc: true, document: "[]", expression: `first`, expected: []string{}, }, { description: "No filter on scalar returns nothing", skipDoc: true, document: "hello", expression: `first`, expected: []string{}, }, { description: "No filter on null returns nothing", skipDoc: true, document: "null", expression: `first`, expected: []string{}, }, } func TestFirstOperatorScenarios(t *testing.T) { for _, tt := range firstOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "first", firstOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_flatten.go ================================================ package yqlib import ( "fmt" ) type flattenPreferences struct { depth int } func flatten(node *CandidateNode, depth int) { if depth == 0 { return } if node.Kind != SequenceNode { return } content := node.Content newSeq := make([]*CandidateNode, 0) for i := 0; i < len(content); i++ { if content[i].Kind == SequenceNode { flatten(content[i], depth-1) for j := 0; j < len(content[i].Content); j++ { newSeq = append(newSeq, content[i].Content[j]) } } else { newSeq = append(newSeq, content[i]) } } node.Content = make([]*CandidateNode, 0) node.AddChildren(newSeq) } func flattenOp(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("flatten Operator") depth := expressionNode.Operation.Preferences.(flattenPreferences).depth for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return Context{}, fmt.Errorf("only arrays are supported for flatten") } flatten(candidate, depth) } return context, nil } ================================================ FILE: pkg/yqlib/operator_flatten_test.go ================================================ package yqlib import ( "testing" ) var flattenOperatorScenarios = []expressionScenario{ { description: "Flatten", subdescription: "Recursively flattens all arrays", document: `[1, [2], [[3]]]`, expression: `flatten`, expected: []string{ "D0, P[], (!!seq)::[1, 2, 3]\n", }, }, { description: "Flatten splat", skipDoc: true, document: `[1, [2], [[3]]]`, expression: `flatten[]`, expected: []string{ "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!int)::3\n", }, }, { description: "Flatten with depth of one", document: `[1, [2], [[3]]]`, expression: `flatten(1)`, expected: []string{ "D0, P[], (!!seq)::[1, 2, [3]]\n", }, }, { description: "Flatten with depth and splat", skipDoc: true, document: `[1, [2], [[3]]]`, expression: `flatten(1)[]`, expected: []string{ "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!seq)::[3]\n", }, }, { description: "Flatten empty array", document: `[[]]`, expression: `flatten`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { description: "Flatten array of objects", document: `[{foo: bar}, [{foo: baz}]]`, expression: `flatten`, expected: []string{ "D0, P[], (!!seq)::[{foo: bar}, {foo: baz}]\n", }, }, } func TestFlattenOperatorScenarios(t *testing.T) { for _, tt := range flattenOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "flatten", flattenOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_group_by.go ================================================ package yqlib import ( "container/list" "fmt" "github.com/elliotchance/orderedmap" ) func processIntoGroups(d *dataTreeNavigator, context Context, rhsExp *ExpressionNode, node *CandidateNode) (*orderedmap.OrderedMap, error) { var newMatches = orderedmap.NewOrderedMap() for _, child := range node.Content { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), rhsExp) if err != nil { return nil, err } keyValue := "null" if rhs.MatchingNodes.Len() > 0 { first := rhs.MatchingNodes.Front() keyCandidate := first.Value.(*CandidateNode) keyValue = keyCandidate.Value } groupList, exists := newMatches.Get(keyValue) if !exists { groupList = list.New() newMatches.Set(keyValue, groupList) } groupList.(*list.List).PushBack(child) } return newMatches, nil } func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("groupBy Operator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return Context{}, fmt.Errorf("only arrays are supported for group by") } newMatches, err := processIntoGroups(d, context, expressionNode.RHS, candidate) if err != nil { return Context{}, err } resultNode := candidate.CreateReplacement(SequenceNode, "!!seq", "") for groupEl := newMatches.Front(); groupEl != nil; groupEl = groupEl.Next() { groupResultNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} groupList := groupEl.Value.(*list.List) for groupItem := groupList.Front(); groupItem != nil; groupItem = groupItem.Next() { groupResultNode.AddChild(groupItem.Value.(*CandidateNode)) } resultNode.AddChild(groupResultNode) } results.PushBack(resultNode) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_group_by_test.go ================================================ package yqlib import ( "testing" ) var groupByOperatorScenarios = []expressionScenario{ { description: "Group by field", document: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`, expression: `group_by(.foo)`, expected: []string{ "D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", }, }, { description: "Group splat", skipDoc: true, document: `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`, expression: `group_by(.foo)[]`, expected: []string{ "D0, P[0], (!!seq)::- {foo: 1, bar: 10}\n- {foo: 1, bar: 1}\n", "D0, P[1], (!!seq)::- {foo: 3, bar: 100}\n", }, }, { description: "Group by field, with nulls", document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, expression: `group_by(.foo)`, expected: []string{ "D0, P[], (!!seq)::- - {cat: dog}\n - {no: foo for you}\n- - {foo: 1, bar: 10}\n - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n", }, }, } func TestGroupByOperatorScenarios(t *testing.T) { for _, tt := range groupByOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "group-by", groupByOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_has.go ================================================ package yqlib import ( "container/list" "strconv" ) func hasOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("hasOperation") var results = list.New() rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } wantedKey := "null" wanted := &CandidateNode{Tag: "!!null"} if rhs.MatchingNodes.Len() != 0 { wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode) wantedKey = wanted.Value } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) var contents = candidate.Content switch candidate.Kind { case MappingNode: candidateHasKey := false for index := 0; index < len(contents) && !candidateHasKey; index = index + 2 { key := contents[index] if key.Value == wantedKey { candidateHasKey = true } } results.PushBack(createBooleanCandidate(candidate, candidateHasKey)) case SequenceNode: candidateHasKey := false if wanted.Tag == "!!int" { var number, errParsingInt = strconv.ParseInt(wantedKey, 10, 64) if errParsingInt != nil { return Context{}, errParsingInt } candidateHasKey = int64(len(contents)) > number } results.PushBack(createBooleanCandidate(candidate, candidateHasKey)) default: results.PushBack(createBooleanCandidate(candidate, false)) } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_has_test.go ================================================ package yqlib import ( "testing" ) var hasOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `a: hello`, expression: `has("a")`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { skipDoc: true, document: `a: hello`, expression: `has(.b) as $c | .`, expected: []string{ "D0, P[], (!!map)::a: hello\n", }, }, { skipDoc: true, document: `a: hello`, expression: `has(.b)`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { description: "Has map key", document: `- a: "yes" - a: ~ - a: - b: nope `, expression: `.[] | has("a")`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::true\n", "D0, P[2], (!!bool)::true\n", "D0, P[3], (!!bool)::false\n", }, }, { description: "Select, checking for existence of deep paths", subdescription: "Simply pipe in parent expressions into `has`", document: "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}", expression: `.[] | select(.a.b | has("c"))`, expected: []string{ "D0, P[0], (!!map)::{a: {b: {c: cat}}}\n", }, }, { dontFormatInputForDoc: true, description: "Has array index", document: `- [] - [1] - [1, 2] - [1, null] - [1, 2, 3] `, expression: `.[] | has(1)`, expected: []string{ "D0, P[0], (!!bool)::false\n", "D0, P[1], (!!bool)::false\n", "D0, P[2], (!!bool)::true\n", "D0, P[3], (!!bool)::true\n", "D0, P[4], (!!bool)::true\n", }, }, } func TestHasOperatorScenarios(t *testing.T) { for _, tt := range hasOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "has", hasOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_keys.go ================================================ package yqlib import ( "container/list" "fmt" ) func isKeyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("isKeyOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) results.PushBack(createBooleanCandidate(candidate, candidate.IsMapKey)) } return context.ChildContext(results), nil } func getKeyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("getKeyOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Key != nil { results.PushBack(candidate.Key) } } return context.ChildContext(results), nil } func keysOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("keysOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) var targetNode *CandidateNode switch candidate.Kind { case MappingNode: targetNode = getMapKeys(candidate) case SequenceNode: targetNode = getIndices(candidate) default: return Context{}, fmt.Errorf("cannot get keys of %v, keys only works for maps and arrays", candidate.Tag) } results.PushBack(targetNode) } return context.ChildContext(results), nil } func getMapKeys(node *CandidateNode) *CandidateNode { contents := make([]*CandidateNode, 0) for index := 0; index < len(node.Content); index = index + 2 { contents = append(contents, node.Content[index]) } seq := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} seq.AddChildren(contents) return seq } func getIndices(node *CandidateNode) *CandidateNode { var contents = make([]*CandidateNode, len(node.Content)) for index := range node.Content { contents[index] = createScalarNode(index, fmt.Sprintf("%v", index)) } seq := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} seq.AddChildren(contents) return seq } ================================================ FILE: pkg/yqlib/operator_keys_test.go ================================================ package yqlib import ( "testing" ) var expectedIsKey = `D0, P[], (!!seq)::- p: "" isKey: false tag: '!!map' - p: a isKey: true tag: '!!str' - p: a isKey: false tag: '!!map' - p: a.b isKey: true tag: '!!str' - p: a.b isKey: false tag: '!!seq' - p: a.b.0 isKey: false tag: '!!str' - p: a.c isKey: true tag: '!!str' - p: a.c isKey: false tag: '!!str' ` var keysOperatorScenarios = []expressionScenario{ { description: "Map keys", document: `{dog: woof, cat: meow}`, expression: `keys`, expected: []string{ "D0, P[], (!!seq)::- dog\n- cat\n", }, }, { description: "Map keys with splat", skipDoc: true, document: `{dog: woof, cat: meow}`, expression: `keys[]`, expected: []string{ "D0, P[0], (!!str)::dog\n", "D0, P[1], (!!str)::cat\n", }, }, { skipDoc: true, document: `{}`, expression: `keys`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { description: "Array keys", document: `[apple, banana]`, expression: `keys`, expected: []string{ "D0, P[], (!!seq)::- 0\n- 1\n", }, }, { skipDoc: true, document: `[]`, expression: `keys`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { description: "Retrieve array key", document: "[1,2,3]", expression: `.[1] | key`, expected: []string{ "D0, P[1], (!!int)::1\n", }, }, { description: "Retrieve map key", document: "a: thing", expression: `.a | key`, expected: []string{ "D0, P[a], (!!str)::a\n", }, }, { description: "No key", document: "{}", expression: `key`, expected: []string{}, }, { description: "Update map key", document: "a:\n x: 3\n y: 4", expression: `(.a.x | key) = "meow"`, expected: []string{ "D0, P[], (!!map)::a:\n meow: 3\n y: 4\n", }, }, { description: "Get comment from map key", document: "a: \n # comment on key\n x: 3\n y: 4", expression: `.a.x | key | headComment`, expected: []string{ "D0, P[a x], (!!str)::comment on key\n", }, }, { description: "Check node is a key", document: "a: \n b: [cat]\n c: frog\n", expression: `[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]`, expected: []string{ expectedIsKey, }, }, } func TestKeysOperatorScenarios(t *testing.T) { for _, tt := range keysOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "keys", keysOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_kind.go ================================================ package yqlib import ( "container/list" ) func kindToText(kind Kind) string { switch kind { case MappingNode: return "map" case SequenceNode: return "seq" case ScalarNode: return "scalar" case AliasNode: return "alias" default: return "unknown" } } func getKindOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetKindOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!str", kindToText(candidate.Kind)) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_kind_test.go ================================================ package yqlib import ( "testing" ) var kindOperatorScenarios = []expressionScenario{ { description: "Get kind", document: `{a: cat, b: 5, c: 3.2, e: true, f: [], g: {}, h: null}`, expression: `.. | kind`, expected: []string{ "D0, P[], (!!str)::map\n", "D0, P[a], (!!str)::scalar\n", "D0, P[b], (!!str)::scalar\n", "D0, P[c], (!!str)::scalar\n", "D0, P[e], (!!str)::scalar\n", "D0, P[f], (!!str)::seq\n", "D0, P[g], (!!str)::map\n", "D0, P[h], (!!str)::scalar\n", }, }, { description: "Get kind, ignores custom tags", subdescription: "Unlike tag, kind is not affected by custom tags.", document: `{a: !!thing cat, b: !!foo {}, c: !!bar []}`, expression: `.. | kind`, expected: []string{ "D0, P[], (!!str)::map\n", "D0, P[a], (!!str)::scalar\n", "D0, P[b], (!!str)::map\n", "D0, P[c], (!!str)::seq\n", }, }, { description: "Add comments only to scalars", subdescription: "An example of how you can use kind", document: "a:\n b: 5\n c: 3.2\ne: true\nf: []\ng: {}\nh: null", expression: `(.. | select(kind == "scalar")) line_comment = "this is a scalar"`, expected: []string{"D0, P[], (!!map)::a:\n b: 5 # this is a scalar\n c: 3.2 # this is a scalar\ne: true # this is a scalar\nf: []\ng: {}\nh: null # this is a scalar\n"}, }, } func TestKindOperatorScenarios(t *testing.T) { for _, tt := range kindOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "kind", kindOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_length.go ================================================ package yqlib import ( "container/list" "fmt" ) func lengthOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("lengthOperation") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) var length int switch candidate.Kind { case ScalarNode: if candidate.Tag == "!!null" { length = 0 } else { length = len(candidate.Value) } case MappingNode: length = len(candidate.Content) / 2 case SequenceNode: length = len(candidate.Content) default: length = 0 } result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", length)) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_length_test.go ================================================ package yqlib import ( "testing" ) var lengthOperatorScenarios = []expressionScenario{ { description: "String length", subdescription: "returns length of string", document: `{a: cat}`, expression: `.a | length`, expected: []string{ "D0, P[a], (!!int)::3\n", }, }, { description: "null length", document: `{a: null}`, expression: `.a | length`, expected: []string{ "D0, P[a], (!!int)::0\n", }, }, { skipDoc: true, document: `{a: ~}`, expression: `.a | length`, expected: []string{ "D0, P[a], (!!int)::0\n", }, }, { skipDoc: true, document: "# abc\n{a: key no exist}", expression: `.b | length`, expected: []string{ "D0, P[b], (!!int)::0\n", }, }, { description: "Map length", subdescription: "returns number of entries", document: `{a: cat, c: dog}`, expression: `length`, expected: []string{ "D0, P[], (!!int)::2\n", }, }, { description: "Array length", subdescription: "returns number of elements", document: `[2,4,6,8]`, expression: `length`, expected: []string{ "D0, P[], (!!int)::4\n", }, }, } func TestLengthOperatorScenarios(t *testing.T) { for _, tt := range lengthOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "length", lengthOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_line.go ================================================ package yqlib import ( "container/list" "fmt" ) func lineOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("lineOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!int", fmt.Sprintf("%v", candidate.Line)) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_line_test.go ================================================ package yqlib import ( "testing" ) var lineOperatorScenarios = []expressionScenario{ { description: "Returns line of _value_ node", document: "a: cat\nb:\n c: cat", expression: `.b | line`, expected: []string{ "D0, P[b], (!!int)::3\n", }, }, { description: "Returns line of _key_ node", subdescription: "Pipe through the key operator to get the line of the key", document: "a: cat\nb:\n c: cat", expression: `.b | key | line`, expected: []string{ "D0, P[b], (!!int)::2\n", }, }, { description: "First line is 1", document: "a: cat", expression: `.a | line`, expected: []string{ "D0, P[a], (!!int)::1\n", }, }, { description: "No line data is 0", expression: `{"a": "new entry"} | line`, expected: []string{ "D0, P[], (!!int)::0\n", }, }, } func TestLineOperatorScenarios(t *testing.T) { for _, tt := range lineOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "line", lineOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_load.go ================================================ package yqlib import ( "bufio" "container/list" "fmt" "os" ) var LoadYamlPreferences = YamlPreferences{ LeadingContentPreProcessing: false, PrintDocSeparators: true, UnwrapScalar: true, EvaluateTogether: false, } type loadPrefs struct { decoder Decoder } func loadString(filename string) (*CandidateNode, error) { // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside that directory. filebytes, err := os.ReadFile(filename) // #nosec if err != nil { return nil, err } return &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: string(filebytes)}, nil } func loadWithDecoder(filename string, decoder Decoder) (*CandidateNode, error) { if decoder == nil { return nil, fmt.Errorf("could not load %s", filename) } file, err := os.Open(filename) // #nosec if err != nil { return nil, err } reader := bufio.NewReader(file) documents, err := readDocuments(reader, filename, 0, decoder) if err != nil { return nil, err } if documents.Len() == 0 { // return null candidate return &CandidateNode{Kind: ScalarNode, Tag: "!!null"}, nil } else if documents.Len() == 1 { candidate := documents.Front().Value.(*CandidateNode) return candidate, nil } sequenceNode := &CandidateNode{Kind: SequenceNode} for doc := documents.Front(); doc != nil; doc = doc.Next() { sequenceNode.AddChild(doc.Value.(*CandidateNode)) } return sequenceNode, nil } func loadStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("loadString") if ConfiguredSecurityPreferences.DisableFileOps { return Context{}, fmt.Errorf("file operations have been disabled") } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() == nil { return Context{}, fmt.Errorf("filename expression returned nil") } nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode) filename := nameCandidateNode.Value contentsCandidate, err := loadString(filename) if err != nil { return Context{}, fmt.Errorf("failed to load %v: %w", filename, err) } results.PushBack(contentsCandidate) } return context.ChildContext(results), nil } func loadOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("loadOperator") if ConfiguredSecurityPreferences.DisableFileOps { return Context{}, fmt.Errorf("file operations have been disabled") } loadPrefs := expressionNode.Operation.Preferences.(loadPrefs) // need to evaluate the 1st parameter against the context // and return the data accordingly. var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() == nil { return Context{}, fmt.Errorf("filename expression returned nil") } nameCandidateNode := rhs.MatchingNodes.Front().Value.(*CandidateNode) filename := nameCandidateNode.Value contentsCandidate, err := loadWithDecoder(filename, loadPrefs.decoder) if err != nil { return Context{}, fmt.Errorf("failed to load %v: %w", filename, err) } results.PushBack(contentsCandidate) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_load_test.go ================================================ package yqlib import ( "testing" ) var loadScenarios = []expressionScenario{ { skipDoc: true, description: "Load empty file with a comment", expression: `load("../../examples/empty.yaml")`, expected: []string{ "D0, P[], (!!null)::# comment\n", }, }, { skipDoc: true, description: "Load and splat", expression: `load("../../examples/small.yaml")[]`, expected: []string{ "D0, P[a], (!!str)::cat\n", }, }, { skipDoc: true, description: "Load and traverse", expression: `load("../../examples/small.yaml").a`, expected: []string{ "D0, P[a], (!!str)::cat\n", }, }, { skipDoc: true, description: "Load file with a header comment into an array", document: `- "../../examples/small.yaml"`, expression: `.[] |= load(.)`, expected: []string{ "D0, P[], (!!seq)::- # comment\n # about things\n a: cat\n", }, }, { skipDoc: true, description: "Load empty file with no comment", expression: `load("../../examples/empty-no-comment.yaml")`, expected: []string{ "D0, P[], (!!null)::\n", }, }, { skipDoc: true, description: "Load multiple documents", expression: `load("../../examples/multiple_docs_small.yaml")`, expected: []string{ "D0, P[], ()::- a: Easy! as one two three\n- another:\n document: here\n- - 1\n - 2\n", }, }, { description: "Simple example", document: `{myFile: "../../examples/thing.yml"}`, expression: `load(.myFile)`, expected: []string{ "D0, P[], (!!map)::a: apple is included\nb: cool.\n", }, }, { description: "Replace node with referenced file", subdescription: "Note that you can modify the filename in the load operator if needed.", document: `{something: {file: "thing.yml"}}`, expression: `.something |= load("../../examples/" + .file)`, expected: []string{ "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}}\n", }, }, { description: "Replace _all_ nodes with referenced file", subdescription: "Recursively match all the nodes (`..`) and then filter the ones that have a 'file' attribute. ", document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`, expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`, expected: []string{ "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n", }, }, { description: "Replace node with referenced file as string", subdescription: "This will work for any text based file", document: `{something: {file: "thing.yml"}}`, expression: `.something |= load_str("../../examples/" + .file)`, expected: []string{ "D0, P[], (!!map)::{something: \"a: apple is included\\nb: cool.\"}\n", }, }, { requiresFormat: "xml", description: "Load from XML", document: "cool: things", expression: `.more_stuff = load_xml("../../examples/small.xml")`, expected: []string{ "D0, P[], (!!map)::cool: things\nmore_stuff:\n this: is some xml\n", }, }, { description: "Load from Properties", document: "cool: things", expression: `.more_stuff = load_props("../../examples/small.properties")`, expected: []string{ "D0, P[], (!!map)::cool: things\nmore_stuff:\n this:\n is: a properties file\n", }, }, { description: "Merge from properties", subdescription: "This can be used as a convenient way to update a yaml document", document: "this:\n is: from yaml\n cool: ay\n", expression: `. *= load_props("../../examples/small.properties")`, expected: []string{ "D0, P[], (!!map)::this:\n is: a properties file\n cool: ay\n", }, }, { description: "Load from base64 encoded file", document: "cool: things", expression: `.more_stuff = load_base64("../../examples/base64.txt")`, expected: []string{ "D0, P[], (!!map)::cool: things\nmore_stuff: my secret chilli recipe is....\n", }, }, } func TestLoadScenarios(t *testing.T) { for _, tt := range loadScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "load", loadScenarios) } var loadOperatorSecurityDisabledScenarios = []expressionScenario{ { description: "load() operation fails when security is enabled", subdescription: "Use `--security-disable-file-ops` to disable file operations for security.", expression: `load("../../examples/thing.yml")`, expectedError: "file operations have been disabled", }, { description: "load_str() operation fails when security is enabled", subdescription: "Use `--security-disable-file-ops` to disable file operations for security.", expression: `load_str("../../examples/thing.yml")`, expectedError: "file operations have been disabled", }, { description: "load_xml() operation fails when security is enabled", subdescription: "Use `--security-disable-file-ops` to disable file operations for security.", expression: `load_xml("../../examples/small.xml")`, expectedError: "file operations have been disabled", }, { description: "load_props() operation fails when security is enabled", subdescription: "Use `--security-disable-file-ops` to disable file operations for security.", expression: `load_props("../../examples/small.properties")`, expectedError: "file operations have been disabled", }, { description: "load_base64() operation fails when security is enabled", subdescription: "Use `--security-disable-file-ops` to disable file operations for security.", expression: `load_base64("../../examples/base64.txt")`, expectedError: "file operations have been disabled", }, } func TestLoadOperatorSecurityDisabledScenarios(t *testing.T) { // Save original security preferences originalDisableFileOps := ConfiguredSecurityPreferences.DisableFileOps defer func() { ConfiguredSecurityPreferences.DisableFileOps = originalDisableFileOps }() // Test that load operations fail when DisableFileOps is true ConfiguredSecurityPreferences.DisableFileOps = true for _, tt := range loadOperatorSecurityDisabledScenarios { testScenario(t, &tt) } appendOperatorDocumentScenario(t, "load", loadOperatorSecurityDisabledScenarios) } ================================================ FILE: pkg/yqlib/operator_map.go ================================================ package yqlib import ( "container/list" ) func mapValuesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) //run expression against entries // splat toEntries and pipe it into Rhs splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) if err != nil { return Context{}, err } assignUpdateExp := &ExpressionNode{ Operation: &Operation{OperationType: assignOpType, UpdateAssign: true}, RHS: expressionNode.RHS, } _, err = assignUpdateOperator(d, splatted, assignUpdateExp) if err != nil { return Context{}, err } } return context, nil } func mapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) //run expression against entries // splat toEntries and pipe it into Rhs splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{}) if err != nil { return Context{}, err } if splatted.MatchingNodes.Len() == 0 { results.PushBack(candidate.Copy()) continue } result, err := d.GetMatchingNodes(splatted, expressionNode.RHS) log.Debug("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType) log.Debug("result %v", result) if err != nil { return Context{}, err } selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} collected, err := collectTogether(d, result, selfExpression) if err != nil { return Context{}, err } collected.Style = candidate.Style results.PushBack(collected) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_map_test.go ================================================ package yqlib import ( "testing" ) var mapOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `[1,2,3]`, document2: `[5,6,7]`, expression: `map(. + 1)`, expected: []string{ "D0, P[], (!!seq)::[2, 3, 4]\n", "D0, P[], (!!seq)::[6, 7, 8]\n", }, }, { description: "mapping against an empty array should do nothing", skipDoc: true, document: `[]`, document2: `["cat"]`, expression: `map(3)`, expected: []string{ "D0, P[], (!!seq)::[]\n", "D0, P[], (!!seq)::[3]\n", }, }, { description: "mapping against an empty array should do nothing", skipDoc: true, document: `[[], [5]]`, expression: `.[] |= map(3)`, expected: []string{ "D0, P[], (!!seq)::[[], [3]]\n", }, }, { description: "mapping against an empty array should do nothing #2", skipDoc: true, document: `[]`, document2: `[5]`, expression: `map(3 + .)`, expected: []string{ "D0, P[], (!!seq)::[]\n", "D0, P[], (!!seq)::[8]\n", }, }, { description: "mapping against an empty array should do nothing", skipDoc: true, document: `[[], [5]]`, expression: `.[] |= map(3 + .)`, expected: []string{ "D0, P[], (!!seq)::[[], [8]]\n", }, }, { skipDoc: true, expression: `[] | map(. + 42)`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: `[1,2]`, expression: `map(. + 1)[]`, expected: []string{ "D0, P[0], (!!int)::2\n", "D0, P[1], (!!int)::3\n", }, }, { description: "Map array", document: `[1,2,3]`, expression: `map(. + 1)`, expected: []string{ "D0, P[], (!!seq)::[2, 3, 4]\n", }, }, { skipDoc: true, document: `{}`, document2: `{b: 12}`, expression: `map_values(3)`, expected: []string{ "D0, P[], (!!map)::{}\n", "D0, P[], (!!map)::{b: 3}\n", }, }, { skipDoc: true, document: `{}`, document2: `{b: 12}`, expression: `map_values(3 + .)`, expected: []string{ "D0, P[], (!!map)::{}\n", "D0, P[], (!!map)::{b: 15}\n", }, }, { skipDoc: true, document: `{a: 1, b: 2, c: 3}`, document2: `{x: 10, y: 20, z: 30}`, expression: `map_values(. + 1)`, expected: []string{ "D0, P[], (!!map)::{a: 2, b: 3, c: 4}\n", "D0, P[], (!!map)::{x: 11, y: 21, z: 31}\n", }, }, { description: "map values splat", skipDoc: true, document: `{a: 1, b: 2}`, expression: `map_values(. + 1)[]`, expected: []string{ "D0, P[a], (!!int)::2\n", "D0, P[b], (!!int)::3\n", }, }, { description: "Map object values", document: `{a: 1, b: 2, c: 3}`, expression: `map_values(. + 1)`, expected: []string{ "D0, P[], (!!map)::{a: 2, b: 3, c: 4}\n", }, }, } func TestMapOperatorScenarios(t *testing.T) { for _, tt := range mapOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "map", mapOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_modulo.go ================================================ package yqlib import ( "fmt" "math" "strconv" "strings" ) func moduloOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Modulo operator") return crossFunction(d, context.ReadOnlyClone(), expressionNode, modulo, false) } func modulo(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if lhs.Tag == "!!null" { return nil, fmt.Errorf("%v (%v) cannot modulo by %v (%v)", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath()) } target := lhs.CopyWithoutContent() if lhs.Kind == ScalarNode && rhs.Kind == ScalarNode { if err := moduloScalars(target, lhs, rhs); err != nil { return nil, err } } else { return nil, fmt.Errorf("%v (%v) cannot modulo by %v (%v)", lhs.Tag, lhs.GetNicePath(), rhs.Tag, rhs.GetNicePath()) } return target, nil } func moduloScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { lhsTag := lhs.Tag rhsTag := rhs.guessTagFromCustomType() lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() lhsIsCustom = true } if lhsTag == "!!int" && rhsTag == "!!int" { target.Kind = ScalarNode target.Style = lhs.Style format, lhsNum, err := parseInt64(lhs.Value) if err != nil { return err } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { return err } if rhsNum == 0 { return fmt.Errorf("cannot modulo by 0") } remainder := lhsNum % rhsNum target.Tag = lhs.Tag target.Value = fmt.Sprintf(format, remainder) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { target.Kind = ScalarNode target.Style = lhs.Style lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return err } remainder := math.Mod(lhsNum, rhsNum) if lhsIsCustom { target.Tag = lhs.Tag } else { target.Tag = "!!float" } target.Value = fmt.Sprintf("%v", remainder) } else { return fmt.Errorf("%v cannot modulo by %v", lhsTag, rhsTag) } return nil } ================================================ FILE: pkg/yqlib/operator_modulo_test.go ================================================ package yqlib import ( "testing" ) var moduloOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `[{a: 2.5, b: 2}, {a: 2, b: 0.75}]`, expression: ".[] | .a % .b", expected: []string{ "D0, P[0 a], (!!float)::0.5\n", "D0, P[1 a], (!!float)::0.5\n", }, }, { skipDoc: true, document: `{}`, expression: "(.a / .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { description: "Number modulo - int", subdescription: "If the lhs and rhs are ints then the expression will be calculated with ints.", document: `{a: 13, b: 2}`, expression: `.a = .a % .b`, expected: []string{ "D0, P[], (!!map)::{a: 1, b: 2}\n", }, }, { description: "Number modulo - float", subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", document: `{a: 12, b: 2.5}`, expression: `.a = .a % .b`, expected: []string{ "D0, P[], (!!map)::{a: !!float 2, b: 2.5}\n", }, }, { description: "Number modulo - int by zero", subdescription: "If the lhs is an int and rhs is a 0 the result is an error.", document: `{a: 1, b: 0}`, expression: `.a = .a % .b`, expectedError: "cannot modulo by 0", }, { description: "Number modulo - float by zero", subdescription: "If the lhs is a float and rhs is a 0 the result is NaN.", document: `{a: 1.1, b: 0}`, expression: `.a = .a % .b`, expected: []string{ "D0, P[], (!!map)::{a: !!float NaN, b: 0}\n", }, }, { skipDoc: true, description: "Custom types: that are really numbers", document: "a: !horse 333.975\nb: !goat 299.2", expression: `.a = .a % .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 34.775000000000034\nb: !goat 299.2\n", }, }, { skipDoc: true, document: "a: 2\nb: !goat 2.3", expression: `.a = .a % .b`, expected: []string{ "D0, P[], (!!map)::a: !!float 2\nb: !goat 2.3\n", }, }, { skipDoc: true, description: "Keep anchors", document: "a: &horse [1]", expression: `.a[1] = .a[0] % 2`, expected: []string{ "D0, P[], (!!map)::a: &horse [1, 1]\n", }, }, { skipDoc: true, description: "Modulo int by string", document: "a: 123\nb: '2'", expression: `.a % .b`, expectedError: "!!int cannot modulo by !!str", }, { skipDoc: true, description: "Modulo string by int", document: "a: 2\nb: '123'", expression: `.b % .a`, expectedError: "!!str cannot modulo by !!int", }, { skipDoc: true, description: "Modulo map by int", document: "a: {\"a\":1}\nb: 2", expression: `.a % .b`, expectedError: "!!map (a) cannot modulo by !!int (b)", }, { skipDoc: true, description: "Modulo array by str", document: "a: [1,2]\nb: '2'", expression: `.a % .b`, expectedError: "!!seq (a) cannot modulo by !!str (b)", }, } func TestModuloOperatorScenarios(t *testing.T) { for _, tt := range moduloOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "modulo", moduloOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_multiply.go ================================================ package yqlib import ( "container/list" "fmt" "strconv" "strings" ) type multiplyPreferences struct { AppendArrays bool DeepMergeArrays bool TraversePrefs traversePreferences AssignPrefs assignPreferences } func createMultiplyOp(prefs interface{}) func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { return func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { return &ExpressionNode{Operation: &Operation{OperationType: multiplyOpType, Preferences: prefs}, LHS: lhs, RHS: rhs} } } func multiplyAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var multiplyPrefs = expressionNode.Operation.Preferences return compoundAssignFunction(d, context, expressionNode, createMultiplyOp(multiplyPrefs)) } func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("MultiplyOperator") return crossFunction(d, context.ReadOnlyClone(), expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false) } func getComments(lhs *CandidateNode, rhs *CandidateNode) (leadingContent string, headComment string, footComment string) { leadingContent = rhs.LeadingContent headComment = rhs.HeadComment footComment = rhs.FootComment if lhs.HeadComment != "" || lhs.LeadingContent != "" { headComment = lhs.HeadComment leadingContent = lhs.LeadingContent } if lhs.FootComment != "" { footComment = lhs.FootComment } return leadingContent, headComment, footComment } func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { // need to do this before unWrapping the potential document node leadingContent, headComment, footComment := getComments(lhs, rhs) log.Debugf("Multiplying LHS: %v", NodeToString(lhs)) log.Debugf("- RHS: %v", NodeToString(rhs)) if rhs.Tag == "!!null" { return lhs.Copy(), nil } if (lhs.Kind == MappingNode && rhs.Kind == MappingNode) || (lhs.Tag == "!!null" && rhs.Kind == MappingNode) || (lhs.Kind == SequenceNode && rhs.Kind == SequenceNode) || (lhs.Tag == "!!null" && rhs.Kind == SequenceNode) { var newBlank = lhs.Copy() newBlank.LeadingContent = leadingContent newBlank.HeadComment = headComment newBlank.FootComment = footComment return mergeObjects(d, context.WritableClone(), newBlank, rhs, preferences) } return multiplyScalars(lhs, rhs) } } func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { lhsTag := lhs.Tag rhsTag := rhs.guessTagFromCustomType() lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() lhsIsCustom = true } if lhsTag == "!!int" && rhsTag == "!!int" { return multiplyIntegers(lhs, rhs) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { return multiplyFloats(lhs, rhs, lhsIsCustom) } else if (lhsTag == "!!str" && rhsTag == "!!int") || (lhsTag == "!!int" && rhsTag == "!!str") { return repeatString(lhs, rhs) } return nil, fmt.Errorf("cannot multiply %v with %v", lhs.Tag, rhs.Tag) } func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode, lhsIsCustom bool) (*CandidateNode, error) { target := lhs.CopyWithoutContent() target.Kind = ScalarNode target.Style = lhs.Style if lhsIsCustom { target.Tag = lhs.Tag } else { target.Tag = "!!float" } lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return nil, err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return nil, err } target.Value = fmt.Sprintf("%v", lhsNum*rhsNum) return target, nil } func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { target := lhs.CopyWithoutContent() target.Kind = ScalarNode target.Style = lhs.Style target.Tag = lhs.Tag format, lhsNum, err := parseInt64(lhs.Value) if err != nil { return nil, err } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { return nil, err } target.Value = fmt.Sprintf(format, lhsNum*rhsNum) return target, nil } func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { var stringNode *CandidateNode var intNode *CandidateNode if lhs.Tag == "!!str" { stringNode = lhs intNode = rhs } else { stringNode = rhs intNode = lhs } target := lhs.CopyWithoutContent() target.UpdateAttributesFrom(stringNode, assignPreferences{}) count, err := parseInt(intNode.Value) if err != nil { return nil, err } else if count < 0 { return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count) } else if count > 10000000 { return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count) } target.Value = strings.Repeat(stringNode.Value, count) return target, nil } func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) { var results = list.New() // only need to recurse the array if we are doing a deep merge prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays, TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true, ExactKeyMatch: true}} log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays) log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays) err := recursiveDecent(results, context.SingleChildContext(rhs), prefs) if err != nil { return nil, err } var pathIndexToStartFrom int if results.Front() != nil { pathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).GetPath()) log.Debugf("pathIndexToStartFrom: %v", pathIndexToStartFrom) } for el := results.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("going to applied assignment to LHS: %v with RHS: %v", NodeToString(lhs), NodeToString(candidate)) if candidate.Tag == "!!merge" { continue } err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences) if err != nil { return nil, err } log.Debugf("applied assignment to LHS: %v", NodeToString(lhs)) } return lhs, nil } func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error { shouldAppendArrays := preferences.AppendArrays lhsPath := rhs.GetPath()[pathIndexToStartFrom:] log.Debugf("merge - lhsPath %v", lhsPath) assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs} if shouldAppendArrays && rhs.Kind == SequenceNode { assignmentOp.OperationType = addAssignOpType log.Debugf("merge - assignmentOp.OperationType = addAssignOpType") } else if !preferences.DeepMergeArrays && rhs.Kind == SequenceNode || (rhs.Kind == ScalarNode || rhs.Kind == AliasNode) { assignmentOp.OperationType = assignOpType assignmentOp.UpdateAssign = false log.Debugf("merge - rhs.Kind == SequenceNode: %v", rhs.Kind == SequenceNode) log.Debugf("merge - rhs.Kind == ScalarNode: %v", rhs.Kind == ScalarNode) log.Debugf("merge - rhs.Kind == AliasNode: %v", rhs.Kind == AliasNode) log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign") } else { log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}") } rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhs} assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, LHS: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), RHS: &ExpressionNode{Operation: rhsOp}, } _, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode) return err } ================================================ FILE: pkg/yqlib/operator_multiply_test.go ================================================ package yqlib import ( "fmt" "strings" "testing" ) var doc1 = `list: # Hi this is a comment. # Hello this is another comment. - "abc"` var doc2 = `list2: # This is yet another comment. # Indeed this is yet another comment. - "123"` var docExpected = `D0, P[], (!!map)::list: # Hi this is a comment. # Hello this is another comment. - "abc" list2: # This is yet another comment. # Indeed this is yet another comment. - "123" ` var mergeArrayWithAnchors = `sample: - &a - <<: *a ` var mergeArraysObjectKeysText = ` This is a fairly complex expression - you can use it as is by providing the environment variables as seen in the example below. It merges in the array provided in the second file into the first - matching on equal keys. Explanation: The approach, at a high level, is to reduce into a merged map (keyed by the unique key) and then convert that back into an array. First the expression will create a map from the arrays keyed by the idPath, the unique field we want to merge by. The reduce operator is merging '({}; . * $item )', so array elements with the matching key will be merged together. Next, we convert the map back to an array, using reduce again, concatenating all the map values together. Finally, we set the result of the merged array back into the first doc. Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223) ` var mergeExpression = ` ( (( (eval(strenv(originalPath)) + eval(strenv(otherPath))) | .[] | {(eval(strenv(idPath))): .}) as $item ireduce ({}; . * $item )) as $uniqueMap | ( $uniqueMap | to_entries | .[]) as $item ireduce([]; . + $item.value) ) as $mergedArray | select(fi == 0) | (eval(strenv(originalPath))) = $mergedArray ` var docWithHeader = `# here a: apple ` var nodeWithHeader = `node: # here a: apple ` var docNoComments = `b: banana ` var docWithFooter = `a: apple # footer ` var nodeWithFooter = `a: apple # footer` var document = `a: &cat {name: cat} b: {name: dog} c: <<: *cat ` var mergeWithGlobA = ` "**cat": things, "meow**cat": stuff ` var mergeWithGlobB = ` "**cat": newThings, ` var multiplyOperatorScenarios = []expressionScenario{ { description: "multiple should be readonly", skipDoc: true, document: "", expression: ".x |= (root | (.a * .b))", expected: []string{ "D0, P[], ()::x: null\n", }, }, { description: "glob keys are treated as literals when merging", skipDoc: true, document: mergeWithGlobA, document2: mergeWithGlobB, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ "D0, P[], (!!map)::\n\"**cat\": newThings,\n\"meow**cat\": stuff\n", }, }, { skipDoc: true, document: mergeArrayWithAnchors, expression: `. * .`, expected: []string{ "D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n", }, }, { skipDoc: true, document: `[[c], [b]]`, expression: `.[] | . *+ ["a"]`, expected: []string{ "D0, P[0], (!!seq)::[c, a]\n", "D0, P[1], (!!seq)::[b, a]\n", }, }, { skipDoc: true, document: docWithHeader, document2: docNoComments, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ "D0, P[], (!!map)::# here\n\na: apple\nb: banana\n", }, }, { skipDoc: true, document: nodeWithHeader, document2: docNoComments, expression: `(select(fi == 0) | .node) * select(fi == 1)`, expected: []string{ "D0, P[node], (!!map)::# here\na: apple\nb: banana\n", }, }, { skipDoc: true, document: docNoComments, document2: docWithHeader, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ "D0, P[], (!!map)::# here\n\nb: banana\na: apple\n", }, }, { skipDoc: true, document: docNoComments, document2: nodeWithHeader, expression: `select(fi == 0) * (select(fi == 1) | .node)`, expected: []string{ "D0, P[], (!!map)::b: banana\n# here\na: apple\n", }, }, { skipDoc: true, document: docWithFooter, document2: docNoComments, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ "D0, P[], (!!map)::a: apple\nb: banana\n# footer\n", }, }, { skipDoc: true, document: nodeWithFooter, document2: docNoComments, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ // not sure why there's an extra newline *shrug* "D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n", }, }, { description: "doc 2 has footer", skipDoc: true, document: docNoComments, document2: docWithFooter, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ "D0, P[], (!!map)::b: banana\na: apple\n# footer\n", }, }, { description: "Multiply integers", document: "a: 3\nb: 4", expression: `.a *= .b`, expected: []string{ "D0, P[], (!!map)::a: 12\nb: 4\n", }, }, { description: "Multiply string node X int", document: docNoComments, expression: ".b * 4", expected: []string{ fmt.Sprintf("D0, P[b], (!!str)::%s\n", strings.Repeat("banana", 4)), }, }, { description: "Multiply int X string node", document: docNoComments, expression: "4 * .b", expected: []string{ fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)), }, }, { description: "Multiply string X int node", document: `n: 4 `, expression: `"banana" * .n`, expected: []string{ fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)), }, }, { description: "Multiply string X by negative int", skipDoc: true, document: `n: -4`, expression: `"banana" * .n`, expectedError: "cannot repeat string by a negative number (-4)", }, { description: "Multiply string X by more than 100 million", // very large string.repeats causes a panic skipDoc: true, document: `n: 100000001`, expression: `"banana" * .n`, expectedError: "cannot repeat string by more than 100 million (100000001)", }, { description: "Multiply int node X string", document: `n: 4 `, expression: `.n * "banana"`, expected: []string{ fmt.Sprintf("D0, P[n], (!!str)::%s\n", strings.Repeat("banana", 4)), }, }, { skipDoc: true, document: doc1, document2: doc2, expression: `select(fi == 0) * select(fi == 1)`, expected: []string{ docExpected, }, }, { skipDoc: true, expression: `.x = {"things": "whatever"} * {}`, expected: []string{ "D0, P[], ()::x:\n things: whatever\n", }, }, { skipDoc: true, expression: `.x = {} * {"things": "whatever"}`, expected: []string{ "D0, P[], ()::x:\n things: whatever\n", }, }, { skipDoc: true, expression: `3 * 4.5`, expected: []string{ "D0, P[], (!!float)::13.5\n", }, }, { skipDoc: true, expression: `4.5 * 3`, expected: []string{ "D0, P[], (!!float)::13.5\n", }, }, { skipDoc: true, document: `{a: {also: [1]}, b: {also: me}}`, expression: `. * {"a" : .b}`, expected: []string{ "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", }, }, { skipDoc: true, document: "# b\nb:\n # a\n a: cat", expression: "{} * .", expected: []string{ "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", }, }, { skipDoc: true, document: "# b\nb:\n # a\n a: cat", expression: ". * {}", expected: []string{ "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", }, }, { skipDoc: true, document: `{a: &a { b: &b { c: &c cat } } }`, expression: `{} * .`, expected: []string{ "D0, P[], (!!map)::a: &a\n b: &b\n c: &c cat\n", }, }, { skipDoc: true, document: `{a: 2, b: 5}`, document2: `{a: 3, b: 10}`, expression: `.a * .b`, expected: []string{ "D0, P[a], (!!int)::10\n", "D0, P[a], (!!int)::20\n", "D0, P[a], (!!int)::15\n", "D0, P[a], (!!int)::30\n", }, }, { skipDoc: true, document: `{a: 2}`, document2: `{b: 10}`, expression: `select(fi ==0) * select(fi==1)`, expected: []string{ "D0, P[], (!!map)::{a: 2, b: 10}\n", }, }, { skipDoc: true, expression: `{} * {"cat":"dog"}`, expected: []string{ "D0, P[], (!!map)::cat: dog\n", }, }, { skipDoc: true, document: `{a: {also: me}, b: {also: [1]}}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", }, }, { description: "Merge objects together, returning merged result only", document: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`, expression: `.a * .b`, expected: []string{ "D0, P[a], (!!map)::{field: {g: wizz}, fieldA: cat, fieldB: dog}\n", }, }, { description: "Merge objects together, returning parent object", document: `{a: {field: me, fieldA: cat}, b: {field: {g: wizz}, fieldB: dog}}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: {field: {g: wizz}, fieldA: cat, fieldB: dog}, b: {field: {g: wizz}, fieldB: dog}}\n", }, }, { skipDoc: true, document: `{a: {also: {g: wizz}}, b: {also: me}}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", }, }, { skipDoc: true, document: `{a: {also: {g: wizz}}, b: {also: [1]}}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n", }, }, { skipDoc: true, document: `{a: {also: [1]}, b: {also: {g: wizz}}}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n", }, }, { skipDoc: true, document: `{a: {things: great}, b: {also: me}}`, expression: `. * {"a": .b}`, expected: []string{ "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", }, }, { description: "Merge keeps style of LHS", dontFormatInputForDoc: true, document: "a: {things: great}\nb:\n also: \"me\"", expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::a: {things: great, also: \"me\"}\nb:\n also: \"me\"\n", }, }, { description: "Merge arrays", document: `{a: [1,2,3], b: [3,4,5]}`, expression: `. * {"a":.b}`, expected: []string{ "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n", }, }, { skipDoc: true, document: `{a: [1], b: [2]}`, expression: `.a *+ .b`, expected: []string{ "D0, P[a], (!!seq)::[1, 2]\n", }, }, { description: "Merge, only existing fields", document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, expression: `.a *? .b`, expected: []string{ "D0, P[a], (!!map)::{thing: two, cat: frog}\n", }, }, { description: "Merge, only new fields", document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, expression: `.a *n .b`, expected: []string{ "D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n", }, }, { skipDoc: true, document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, expression: `.a *?d .b`, expected: []string{ "D0, P[a], (!!seq)::[{thing: two}]\n", }, }, { skipDoc: true, document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, expression: `.a *nd .b`, expected: []string{ "D0, P[a], (!!seq)::[{thing: one, missing: two}]\n", }, }, { skipDoc: true, document: `{a: {array: [1]}, b: {}}`, expression: `.b *+ .a`, expected: []string{ "D0, P[b], (!!map)::array: [1]\n", }, }, { description: "Merge, appending arrays", document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`, expression: `.a *+ .b`, expected: []string{ "D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n", }, }, { description: "Merge, only existing fields, appending arrays", document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, expression: `.a *?+ .b`, expected: []string{ "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n", }, }, { description: "Merge, only new fields, appending arrays", subdescription: "Append (+) with (n) has no effect.", skipDoc: true, document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, expression: `.a *n+ .b`, expected: []string{ "D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n", }, }, { description: "Merge, deeply merging arrays", subdescription: "Merging arrays deeply means arrays are merged like objects, with indices as their key. In this case, we merge the first item in the array and do nothing with the second.", document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`, expression: `.a *d .b`, expected: []string{ "D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n", }, }, { description: "Merge arrays of objects together, matching on a key", subdescription: mergeArraysObjectKeysText, document: `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`, document2: `newArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`, environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"}, expression: mergeExpression, expected: []string{ "D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n", }, }, { description: "Merge to prefix an element", document: `{a: cat, b: dog}`, expression: `. * {"a": {"c": .a}}`, expected: []string{ "D0, P[], (!!map)::{a: {c: cat}, b: dog}\n", }, }, { description: "Merge with simple aliases", document: `{a: &cat {c: frog}, b: {f: *cat}, c: {g: thongs}}`, expression: `.c * .b`, expected: []string{ "D0, P[c], (!!map)::{g: thongs, f: *cat}\n", }, }, { description: "Merge copies anchor names", document: `{a: {c: &cat frog}, b: {f: *cat}, c: {g: thongs}}`, expression: `.c * .a`, expected: []string{ "D0, P[c], (!!map)::{g: thongs, c: &cat frog}\n", }, }, { description: "Merge with merge anchors", document: mergeDocSample, expression: `.foobar * .foobarList`, expected: []string{ "D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n", }, }, { skipDoc: true, document: document, expression: `.b * .c`, expected: []string{ "D0, P[b], (!!map)::{name: dog, <<: *cat}\n", }, }, { description: "Custom types: that are really numbers", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse 2\nb: !goat 3", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: !horse 6\nb: !goat 3\n", }, }, { skipDoc: true, description: "Custom types: that are really numbers", document: "a: !horse 2.5\nb: !goat 3.5", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: !horse 8.75\nb: !goat 3.5\n", }, }, { skipDoc: true, description: "Custom types: that are really numbers", document: "a: 2\nb: !goat 3.5", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: !!float 7\nb: !goat 3.5\n", }, }, { skipDoc: true, description: "Custom types: that are really arrays", document: "a: !horse [1,2]\nb: !goat [3]", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: !horse [3]\nb: !goat [3]\n", }, }, { description: "Custom types: that are really maps", subdescription: "Custom tags will be maintained.", document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", }, }, { description: "Custom types: clobber tags", subdescription: "Use the `c` option to clobber custom tags. Note that the second tag is now used.", document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", expression: ".a *=c .b", expected: []string{ "D0, P[], (!!map)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", }, }, { skipDoc: true, description: "Custom types: clobber tags - *=", subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used.", document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", expression: ".a =c .a *c .b", expected: []string{ "D0, P[], (!!map)::a: !goat {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", }, }, { skipDoc: true, description: "Custom types: dont clobber tags - *=", subdescription: "Use the `c` option to clobber custom tags - on both the `=` and `*` operator. Note that the second tag is now used.", document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", expression: ".a *= .b", expected: []string{ "D0, P[], (!!map)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", }, }, { skipDoc: true, description: "Custom types: that are really maps", document: "a: {cat: !horse meow}\nb: {cat: 5}", expression: ".a = .a * .b", expected: []string{ "D0, P[], (!!map)::a: {cat: !horse 5}\nb: {cat: 5}\n", }, }, { skipDoc: true, description: "Relative merge, new fields only", document: "a: {a: original}\n", expression: `.a *=n load("../../examples/thing.yml")`, expected: []string{ "D0, P[], (!!map)::a: {a: original, b: cool.}\n", }, }, { skipDoc: true, description: "Relative merge", document: "a: {a: original}\n", expression: `.a *= load("../../examples/thing.yml")`, expected: []string{ "D0, P[], (!!map)::a: {a: apple is included, b: cool.}\n", }, }, { description: "Merging a null with a map", expression: `null * {"some": "thing"}`, expected: []string{ "D0, P[], (!!map)::some: thing\n", }, }, { description: "Merging a map with null", expression: `{"some": "thing"} * null`, expected: []string{ "D0, P[], (!!map)::some: thing\n", }, }, { description: "Merging a null with an array", expression: `null * ["some"]`, expected: []string{ "D0, P[], (!!seq)::- some\n", }, }, { description: "Merging an array with null", expression: `["some"] * null`, expected: []string{ "D0, P[], (!!seq)::- some\n", }, }, { skipDoc: true, expression: `null * null`, expected: []string{ "D0, P[], (!!null)::null\n", }, }, } func TestMultiplyOperatorScenarios(t *testing.T) { for _, tt := range multiplyOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "multiply-merge", multiplyOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_omit.go ================================================ package yqlib import ( "container/list" "strconv" ) func omitMap(original *CandidateNode, indices *CandidateNode) *CandidateNode { filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)*2)) for index := 0; index < len(original.Content); index += 2 { pos := findInArray(indices, original.Content[index]) if pos < 0 { clonedKey := original.Content[index].Copy() clonedValue := original.Content[index+1].Copy() filteredContent = append(filteredContent, clonedKey, clonedValue) } } result := original.CopyWithoutContent() result.AddChildren(filteredContent) return result } func omitSequence(original *CandidateNode, indices *CandidateNode) *CandidateNode { filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content))) for index := 0; index < len(original.Content); index++ { pos := findInArray(indices, createScalarNode(index, strconv.Itoa(index))) if pos < 0 { filteredContent = append(filteredContent, original.Content[index].Copy()) } } result := original.CopyWithoutContent() result.AddChildren(filteredContent) return result } func omitOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Omit") contextIndicesToOmit, err := d.GetMatchingNodes(context, expressionNode.RHS) if err != nil { return Context{}, err } indicesToOmit := &CandidateNode{} if contextIndicesToOmit.MatchingNodes.Len() > 0 { indicesToOmit = contextIndicesToOmit.MatchingNodes.Front().Value.(*CandidateNode) } if len(indicesToOmit.Content) == 0 { log.Debugf("No omit indices specified") return context, nil } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) var replacement *CandidateNode switch node.Kind { case MappingNode: replacement = omitMap(node, indicesToOmit) case SequenceNode: replacement = omitSequence(node, indicesToOmit) default: log.Debugf("Omit from type %v (%v) is noop", node.Tag, node.GetNicePath()) return context, nil } replacement.LeadingContent = node.LeadingContent results.PushBack(replacement) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_omit_test.go ================================================ package yqlib import ( "testing" ) var omitOperatorScenarios = []expressionScenario{ { description: "Omit keys from map", subdescription: "Note that non existent keys are skipped.", document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= omit(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::myMap: {dog: bark, thing: hamster}\n", }, }, { description: "Omit splat", skipDoc: true, document: "{cat: meow, dog: bark, hamster: squeak}\n", expression: `omit(["dog"])[]`, expected: []string{ "D0, P[cat], (!!str)::meow\n", "D0, P[hamster], (!!str)::squeak\n", }, }, { description: "Omit keys from map", skipDoc: true, document: "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= omit(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::!things myMap: {dog: bark, thing: hamster}\n", }, }, { description: "Omit keys from map with comments", skipDoc: true, document: "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n# xyz\n", expression: `.myMap |= omit(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::# abc\nmyMap: {dog: bark, thing: hamster}\n# xyz\n", }, }, { description: "Omit indices from array", subdescription: "Note that non existent indices are skipped.", document: `[cat, leopard, lion]`, expression: `omit([2, 0, 734, -5])`, expected: []string{ "D0, P[], (!!seq)::[leopard]\n", }, }, { description: "Omit indices from array with comments", skipDoc: true, document: "# abc\n[cat, leopard, lion]\n# xyz", expression: `omit([2, 0, 734, -5])`, expected: []string{ "D0, P[], (!!seq)::# abc\n[leopard]\n# xyz\n", }, }, } func TestOmitOperatorScenarios(t *testing.T) { for _, tt := range omitOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "omit", omitOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_parent.go ================================================ package yqlib import "container/list" type parentOpPreferences struct { Level int } func getParentsOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("getParentsOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) parentsList := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"} parent := candidate.Parent for parent != nil { parentsList.AddChild(parent) parent = parent.Parent } results.PushBack(parentsList) } return context.ChildContext(results), nil } func getParentOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("getParentOperator") var results = list.New() prefs := expressionNode.Operation.Preferences.(parentOpPreferences) for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) // Handle negative levels: count total parents first levelsToGoUp := prefs.Level if prefs.Level < 0 { // Count all parents totalParents := 0 temp := candidate.Parent for temp != nil { totalParents++ temp = temp.Parent } // Convert negative index to positive // -1 means last parent (root), -2 means second to last, etc. levelsToGoUp = totalParents + prefs.Level + 1 if levelsToGoUp < 0 { levelsToGoUp = 0 } } currentLevel := 0 for currentLevel < levelsToGoUp && candidate != nil { log.Debugf("currentLevel: %v, desired: %v", currentLevel, levelsToGoUp) log.Debugf("candidate: %v", NodeToString(candidate)) candidate = candidate.Parent currentLevel++ } log.Debugf("found candidate: %v", NodeToString(candidate)) if candidate != nil { results.PushBack(candidate) } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_parent_test.go ================================================ package yqlib import ( "testing" ) var parentOperatorScenarios = []expressionScenario{ { description: "Simple example", document: `a: {nested: cat}`, expression: `.a.nested | parent`, expected: []string{ "D0, P[a], (!!map)::{nested: cat}\n", }, }, { description: "Parent of nested matches", document: `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`, expression: `.. | select(. == "banana") | parent`, expected: []string{ "D0, P[b], (!!map)::{fruit: banana, name: sam}\n", }, }, { description: "Get parent attribute", document: `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`, expression: `.. | select(. == "banana") | parent.name`, expected: []string{ "D0, P[b name], (!!str)::sam\n", }, }, { description: "Get parents", subdescription: "Match all parents", document: "{a: {b: {c: cat} } }", expression: `.a.b.c | parents`, expected: []string{ "D0, P[], (!!seq)::- {c: cat}\n- {b: {c: cat}}\n- {a: {b: {c: cat}}}\n", }, }, { description: "Get the top (root) parent", subdescription: "Use negative numbers to get the top parents. You can think of this as indexing into the 'parents' array above", document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(-1)`, expected: []string{ "D0, P[], (!!map)::a:\n b:\n c: cat\n", }, }, { description: "Root", subdescription: "Alias for parent(-1), returns the top level parent. This is usually the document node.", document: "a:\n b:\n c: cat\n", expression: `.a.b.c | root`, expected: []string{ "D0, P[], (!!map)::a:\n b:\n c: cat\n", }, }, { description: "boundary negative", skipDoc: true, document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(-3)`, expected: []string{ "D0, P[a b], (!!map)::c: cat\n", }, }, { description: "large negative", skipDoc: true, document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(-10)`, expected: []string{ "D0, P[a b c], (!!str)::cat\n", }, }, { description: "parent zero", skipDoc: true, document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(0)`, expected: []string{ "D0, P[a b c], (!!str)::cat\n", }, }, { description: "large positive", skipDoc: true, document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(10)`, expected: []string{}, }, { description: "N-th parent", subdescription: "You can optionally supply the number of levels to go up for the parent, the default being 1.", document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(2)`, expected: []string{ "D0, P[a], (!!map)::b:\n c: cat\n", }, }, { description: "N-th parent - another level", document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(3)`, expected: []string{ "D0, P[], (!!map)::a:\n b:\n c: cat\n", }, }, { description: "N-th negative", subdescription: "Similarly, use negative numbers to index backwards from the parents array", document: "a:\n b:\n c: cat\n", expression: `.a.b.c | parent(-2)`, expected: []string{ "D0, P[a], (!!map)::b:\n c: cat\n", }, }, { description: "No parent", document: `{}`, expression: `parent`, expected: []string{}, }, } func TestParentOperatorScenarios(t *testing.T) { for _, tt := range parentOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "parent", parentOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_path.go ================================================ package yqlib import ( "container/list" "fmt" ) func createPathNodeFor(pathElement interface{}) *CandidateNode { switch pathElement := pathElement.(type) { case string: return &CandidateNode{Kind: ScalarNode, Value: pathElement, Tag: "!!str"} default: return &CandidateNode{Kind: ScalarNode, Value: fmt.Sprintf("%v", pathElement), Tag: "!!int"} } } func getPathArrayFromNode(funcName string, node *CandidateNode) ([]interface{}, error) { if node.Kind != SequenceNode { return nil, fmt.Errorf("%v: expected path array, but got %v instead", funcName, node.Tag) } path := make([]interface{}, len(node.Content)) for i, childNode := range node.Content { switch childNode.Tag { case "!!str": path[i] = childNode.Value case "!!int": number, err := parseInt(childNode.Value) if err != nil { return nil, fmt.Errorf("%v: could not parse %v as an int: %w", funcName, childNode.Value, err) } path[i] = number default: return nil, fmt.Errorf("%v: expected either a !!str or !!int in the path, found %v instead", funcName, childNode.Tag) } } return path, nil } // SETPATH(pathArray; value) func setPathOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("SetPath") if expressionNode.RHS.Operation.OperationType != blockOpType { return Context{}, fmt.Errorf("SETPATH must be given a block (;), got %v instead", expressionNode.RHS.Operation.OperationType.Type) } lhsPathContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS.LHS) if err != nil { return Context{}, err } if lhsPathContext.MatchingNodes.Len() != 1 { return Context{}, fmt.Errorf("SETPATH: expected single path but found %v results instead", lhsPathContext.MatchingNodes.Len()) } lhsValue := lhsPathContext.MatchingNodes.Front().Value.(*CandidateNode) lhsPath, err := getPathArrayFromNode("SETPATH", lhsValue) if err != nil { return Context{}, err } lhsTraversalTree := createTraversalTree(lhsPath, traversePreferences{}, false) assignmentOp := &Operation{OperationType: assignOpType} for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) targetContextValue, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS.RHS) if err != nil { return Context{}, err } if targetContextValue.MatchingNodes.Len() != 1 { return Context{}, fmt.Errorf("SETPATH: expected single value on RHS but found %v", targetContextValue.MatchingNodes.Len()) } rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: targetContextValue.MatchingNodes.Front().Value.(*CandidateNode)} assignmentOpNode := &ExpressionNode{ Operation: assignmentOp, LHS: lhsTraversalTree, RHS: &ExpressionNode{Operation: rhsOp}, } _, err = d.GetMatchingNodes(context.SingleChildContext(candidate), assignmentOpNode) if err != nil { return Context{}, err } } return context, nil } func delPathsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("delPaths") // single RHS expression that returns an array of paths (array of arrays) pathArraysContext, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if pathArraysContext.MatchingNodes.Len() != 1 { return Context{}, fmt.Errorf("DELPATHS: expected single value but found %v", pathArraysContext.MatchingNodes.Len()) } pathArraysNode := pathArraysContext.MatchingNodes.Front().Value.(*CandidateNode) if pathArraysNode.Tag != "!!seq" { return Context{}, fmt.Errorf("DELPATHS: expected a sequence of sequences, but found %v", pathArraysNode.Tag) } updatedContext := context for i, child := range pathArraysNode.Content { if child.Tag != "!!seq" { return Context{}, fmt.Errorf("DELPATHS: expected entry [%v] to be a sequence, but its a %v. Note that delpaths takes an array of path arrays, e.g. [[\"a\", \"b\"]]", i, child.Tag) } childPath, err := getPathArrayFromNode("DELPATHS", child) if err != nil { return Context{}, err } childTraversalExp := createTraversalTree(childPath, traversePreferences{}, false) deleteChildOp := &Operation{OperationType: deleteChildOpType} deleteChildOpNode := &ExpressionNode{ Operation: deleteChildOp, RHS: childTraversalExp, } updatedContext, err = d.GetMatchingNodes(updatedContext, deleteChildOpNode) if err != nil { return Context{}, err } } return updatedContext, nil } func getPathOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetPath") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) node := candidate.CreateReplacement(SequenceNode, "!!seq", "") path := candidate.GetPath() content := make([]*CandidateNode, len(path)) for pathIndex := 0; pathIndex < len(path); pathIndex++ { path := path[pathIndex] content[pathIndex] = createPathNodeFor(path) } node.AddChildren(content) results.PushBack(node) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_path_test.go ================================================ package yqlib import ( "testing" ) var documentToPrune = ` parentA: bob parentB: child1: i am child1 child2: i am child2 parentC: child1: me child1 child2: me child2 ` var pathOperatorScenarios = []expressionScenario{ { description: "Map path", document: `{a: {b: cat}}`, expression: `.a.b | path`, expected: []string{ "D0, P[a b], (!!seq)::- a\n- b\n", }, }, { description: "Map path", document: `{a: {b: cat}}`, skipDoc: true, expression: `.a.b | path[]`, expected: []string{ "D0, P[a b 0], (!!str)::a\n", "D0, P[a b 1], (!!str)::b\n", }, }, { skipDoc: true, document: `a: b: c: - 0 - 1 - 2 - 3`, expression: `.a.b.c.[]`, expected: []string{ "D0, P[a b c 0], (!!int)::0\n", "D0, P[a b c 1], (!!int)::1\n", "D0, P[a b c 2], (!!int)::2\n", "D0, P[a b c 3], (!!int)::3\n", }, }, { description: "Get map key", document: `{a: {b: cat}}`, expression: `.a.b | path | .[-1]`, expected: []string{ "D0, P[a b 1], (!!str)::b\n", }, }, { description: "Array path", document: `{a: [cat, dog]}`, expression: `.a.[] | select(. == "dog") | path`, expected: []string{ "D0, P[a 1], (!!seq)::- a\n- 1\n", }, }, { description: "Get array index", document: `{a: [cat, dog]}`, expression: `.a.[] | select(. == "dog") | path | .[-1]`, expected: []string{ "D0, P[a 1 1], (!!int)::1\n", }, }, { description: "Print path and value", document: `{a: [cat, dog, frog]}`, expression: `.a[] | select(. == "*og") | [{"path":path, "value":.}]`, expected: []string{ "D0, P[a 1], (!!seq)::- path:\n - a\n - 1\n value: dog\n", "D0, P[a 2], (!!seq)::- path:\n - a\n - 2\n value: frog\n", }, }, { description: "Set path", document: `{a: {b: cat}}`, expression: `setpath(["a", "b"]; "things")`, expected: []string{ "D0, P[], (!!map)::{a: {b: things}}\n", }, }, { description: "Set on empty document", expression: `setpath(["a", "b"]; "things")`, expected: []string{ "D0, P[], ()::a:\n b: things\n", }, }, { description: "Set path to prune deep paths", subdescription: "Like pick but recursive. This uses `ireduce` to deeply set the selected paths into an empty object.", document: documentToPrune, expression: "(.parentB.child2, .parentC.child1) as $i\n ireduce({}; setpath($i | path; $i))", expected: []string{ "D0, P[], (!!map)::parentB:\n child2: i am child2\nparentC:\n child1: me child1\n", }, }, { description: "Set array path", document: `a: [cat, frog]`, expression: `setpath(["a", 0]; "things")`, expected: []string{ "D0, P[], (!!map)::a: [things, frog]\n", }, }, { description: "Set array path empty", expression: `setpath(["a", 0]; "things")`, expected: []string{ "D0, P[], ()::a:\n - things\n", }, }, { description: "Delete path", subdescription: "Notice delpaths takes an _array_ of paths.", document: `{a: {b: cat, c: dog, d: frog}}`, expression: `delpaths([["a", "c"], ["a", "d"]])`, expected: []string{ "D0, P[], (!!map)::{a: {b: cat}}\n", }, }, { description: "Delete array path", document: `a: [cat, frog]`, expression: `delpaths([["a", 0]])`, expected: []string{ "D0, P[], (!!map)::a: [frog]\n", }, }, { description: "Delete splat", skipDoc: true, document: `a: [cat, frog]`, expression: `delpaths([["a", 0]])[]`, expected: []string{ "D0, P[a], (!!seq)::[frog]\n", }, }, { description: "Delete - wrong parameter", subdescription: "delpaths does not work with a single path array", document: `a: [cat, frog]`, expression: `delpaths(["a", 0])`, expectedError: "DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [[\"a\", \"b\"]]", }, } func TestPathOperatorsScenarios(t *testing.T) { for _, tt := range pathOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "path", pathOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_pick.go ================================================ package yqlib import ( "container/list" "fmt" ) func pickMap(original *CandidateNode, indices *CandidateNode) *CandidateNode { filteredContent := make([]*CandidateNode, 0) for index := 0; index < len(indices.Content); index = index + 1 { keyToFind := indices.Content[index] indexInMap := findKeyInMap(original, keyToFind) if indexInMap > -1 { clonedKey := original.Content[indexInMap].Copy() clonedValue := original.Content[indexInMap+1].Copy() filteredContent = append(filteredContent, clonedKey, clonedValue) } } newNode := original.CopyWithoutContent() newNode.AddChildren(filteredContent) return newNode } func pickSequence(original *CandidateNode, indices *CandidateNode) (*CandidateNode, error) { filteredContent := make([]*CandidateNode, 0) for index := 0; index < len(indices.Content); index = index + 1 { indexInArray, err := parseInt(indices.Content[index].Value) if err != nil { return nil, fmt.Errorf("cannot index array with %v", indices.Content[index].Value) } if indexInArray > -1 && indexInArray < len(original.Content) { filteredContent = append(filteredContent, original.Content[indexInArray].Copy()) } } newNode := original.CopyWithoutContent() newNode.AddChildren(filteredContent) return newNode, nil } func pickOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Pick") contextIndicesToPick, err := d.GetMatchingNodes(context, expressionNode.RHS) if err != nil { return Context{}, err } indicesToPick := &CandidateNode{} if contextIndicesToPick.MatchingNodes.Len() > 0 { indicesToPick = contextIndicesToPick.MatchingNodes.Front().Value.(*CandidateNode) } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) var replacement *CandidateNode switch node.Kind { case MappingNode: replacement = pickMap(node, indicesToPick) case SequenceNode: replacement, err = pickSequence(node, indicesToPick) if err != nil { return Context{}, err } default: return Context{}, fmt.Errorf("cannot pick indices from type %v (%v)", node.Tag, node.GetNicePath()) } replacement.LeadingContent = node.LeadingContent results.PushBack(replacement) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_pick_test.go ================================================ package yqlib import ( "testing" ) var pickOperatorScenarios = []expressionScenario{ { description: "Pick keys from map", subdescription: "Note that the order of the keys matches the pick order and non existent keys are skipped.", document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::myMap: {hamster: squeak, cat: meow}\n", }, }, { description: "Pick keys from map, included all the keys", subdescription: "We create a map of the picked keys plus all the current keys, and run that through unique", document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= pick( (["thing"] + keys) | unique)`, expected: []string{ "D0, P[], (!!map)::myMap: {thing: hamster, cat: meow, dog: bark, hamster: squeak}\n", }, }, { description: "Pick splat", skipDoc: true, document: "{cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `pick(["hamster", "cat"])[]`, expected: []string{ "D0, P[hamster], (!!str)::squeak\n", "D0, P[cat], (!!str)::meow\n", }, }, { description: "Pick keys from map", skipDoc: true, document: "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::!things myMap: {hamster: squeak, cat: meow}\n", }, }, { description: "Pick keys from map with comments", skipDoc: true, document: "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n# xyz\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ "D0, P[], (!!map)::# abc\nmyMap: {hamster: squeak, cat: meow}\n# xyz\n", }, }, { description: "Pick indices from array", subdescription: "Note that the order of the indices matches the pick order and non existent indices are skipped.", document: `[cat, leopard, lion]`, expression: `pick([2, 0, 734, -5])`, expected: []string{ "D0, P[], (!!seq)::[lion, cat]\n", }, }, { description: "Pick indices from array with comments", skipDoc: true, document: "# abc\n[cat, leopard, lion]\n# xyz", expression: `pick([2, 0, 734, -5])`, expected: []string{ "D0, P[], (!!seq)::# abc\n[lion, cat]\n# xyz\n", }, }, } func TestPickOperatorScenarios(t *testing.T) { for _, tt := range pickOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "pick", pickOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_pipe.go ================================================ package yqlib func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if expressionNode.LHS.Operation.OperationType == assignVariableOpType { return variableLoop(d, context, expressionNode) } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } rhsContext := context.ChildContext(lhs.MatchingNodes) rhs, err := d.GetMatchingNodes(rhsContext, expressionNode.RHS) if err != nil { return Context{}, err } return context.ChildContext(rhs.MatchingNodes), nil } ================================================ FILE: pkg/yqlib/operator_pipe_test.go ================================================ package yqlib import ( "testing" ) var pipeOperatorScenarios = []expressionScenario{ { description: "Simple Pipe", document: `{a: {b: cat}}`, expression: `.a | .b`, expected: []string{ "D0, P[a b], (!!str)::cat\n", }, }, { description: "Multiple updates", document: `{a: cow, b: sheep, c: same}`, expression: `.a = "cat" | .b = "dog"`, expected: []string{ "D0, P[], (!!map)::{a: cat, b: dog, c: same}\n", }, }, { skipDoc: true, description: "Don't pass readonly context", expression: `(3 + 4) | ({} | .b = "dog")`, expected: []string{ "D0, P[], (!!map)::b: dog\n", }, }, } func TestPipeOperatorScenarios(t *testing.T) { for _, tt := range pipeOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "pipe", pipeOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_pivot.go ================================================ package yqlib import ( "container/list" "fmt" ) func getUniqueElementTag(seq *CandidateNode) (string, error) { switch l := len(seq.Content); l { case 0: return "", nil default: result := seq.Content[0].Tag for i := 1; i < l; i++ { t := seq.Content[i].Tag if t != result { return "", fmt.Errorf("sequence contains elements of %v and %v types", result, t) } } return result, nil } } var nullNodeFactory = func() *CandidateNode { return createScalarNode(nil, "") } func pad[E any](array []E, length int, factory func() E) []E { sz := len(array) if sz >= length { return array } pad := make([]E, length-sz) for i := 0; i < len(pad); i++ { pad[i] = factory() } return append(array, pad...) } func pivotSequences(seq *CandidateNode) *CandidateNode { sz := len(seq.Content) if sz == 0 { return seq } m := make(map[int][]*CandidateNode) for i := 0; i < sz; i++ { row := seq.Content[i] for j := 0; j < len(row.Content); j++ { e := m[j] if e == nil { e = make([]*CandidateNode, 0, sz) } m[j] = append(pad(e, i, nullNodeFactory), row.Content[j]) } } result := CandidateNode{Kind: SequenceNode} for i := 0; i < len(m); i++ { e := CandidateNode{Kind: SequenceNode} e.AddChildren(pad(m[i], sz, nullNodeFactory)) result.AddChild(&e) } return &result } func pivotMaps(seq *CandidateNode) *CandidateNode { sz := len(seq.Content) if sz == 0 { return &CandidateNode{Kind: MappingNode} } m := make(map[string][]*CandidateNode) keys := make([]string, 0) for i := 0; i < sz; i++ { row := seq.Content[i] for j := 0; j < len(row.Content); j += 2 { k := row.Content[j].Value v := row.Content[j+1] e := m[k] if e == nil { keys = append(keys, k) e = make([]*CandidateNode, 0, sz) } m[k] = append(pad(e, i, nullNodeFactory), v) } } result := CandidateNode{Kind: MappingNode} for _, k := range keys { pivotRow := CandidateNode{Kind: SequenceNode} pivotRow.AddChildren( pad(m[k], sz, nullNodeFactory)) result.AddKeyValueChild(createScalarNode(k, k), &pivotRow) } return &result } func pivotOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debug("Pivot") results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Tag != "!!seq" { return Context{}, fmt.Errorf("cannot pivot node of type %v", candidate.Tag) } tag, err := getUniqueElementTag(candidate) if err != nil { return Context{}, err } var pivot *CandidateNode switch tag { case "!!seq": pivot = pivotSequences(candidate) case "!!map": pivot = pivotMaps(candidate) default: return Context{}, fmt.Errorf("can only pivot elements of !!seq or !!map types, received %v", tag) } results.PushBack(pivot) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_pivot_test.go ================================================ package yqlib import "testing" var pivotOperatorScenarios = []expressionScenario{ { description: "Pivot a sequence of sequences", document: "[[foo, bar, baz], [sis, boom, bah]]\n", expression: `pivot`, expected: []string{ "D0, P[], ()::- - foo\n - sis\n- - bar\n - boom\n- - baz\n - bah\n", }, }, { description: "Pivot splat", skipDoc: true, document: "[[foo, bar], [sis, boom]]\n", expression: `pivot[]`, expected: []string{ "D0, P[0], ()::- foo\n- sis\n", "D0, P[1], ()::- bar\n- boom\n", }, }, { description: "Pivot sequence of heterogeneous sequences", subdescription: `Missing values are "padded" to null.`, document: "[[foo, bar, baz], [sis, boom, bah, blah]]\n", expression: `pivot`, expected: []string{ "D0, P[], ()::- - foo\n - sis\n- - bar\n - boom\n- - baz\n - bah\n- -\n - blah\n", }, }, { description: "Pivot sequence of maps", document: "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z}]\n", expression: `pivot`, expected: []string{ "D0, P[], ()::foo:\n - a\n - x\nbar:\n - b\n - y\nbaz:\n - c\n - z\n", }, }, { description: "Pivot sequence of heterogeneous maps", subdescription: `Missing values are "padded" to null.`, document: "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z, what: ever}]\n", expression: `pivot`, expected: []string{ "D0, P[], ()::foo:\n - a\n - x\nbar:\n - b\n - y\nbaz:\n - c\n - z\nwhat:\n -\n - ever\n", }, }, } func TestPivotOperatorScenarios(t *testing.T) { for _, tt := range pivotOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "pivot", pivotOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_recursive_descent.go ================================================ package yqlib import ( "container/list" ) type recursiveDescentPreferences struct { TraversePreferences traversePreferences RecurseArray bool } func recursiveDescentOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var results = list.New() preferences := expressionNode.Operation.Preferences.(recursiveDescentPreferences) err := recursiveDecent(results, context, preferences) if err != nil { return Context{}, err } return context.ChildContext(results), nil } func recursiveDecent(results *list.List, context Context, preferences recursiveDescentPreferences) error { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("added %v", NodeToString(candidate)) results.PushBack(candidate) if candidate.Kind != AliasNode && len(candidate.Content) > 0 && (preferences.RecurseArray || candidate.Kind != SequenceNode) { children, err := splat(context.SingleChildContext(candidate), preferences.TraversePreferences) if err != nil { return err } err = recursiveDecent(results, children, preferences) if err != nil { return err } } } return nil } ================================================ FILE: pkg/yqlib/operator_recursive_descent_test.go ================================================ package yqlib import ( "testing" ) var recursiveDescentOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{}`, expression: `..`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: `{}`, expression: `...`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: `[]`, expression: `..`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: `[]`, expression: `...`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: `cat`, expression: `..`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { skipDoc: true, document: `cat`, expression: `...`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { description: "Recurse map (values only)", document: `{a: frog}`, expression: `..`, expected: []string{ "D0, P[], (!!map)::{a: frog}\n", "D0, P[a], (!!str)::frog\n", }, }, { description: "Recursively find nodes with keys", subdescription: "Note that this example has wrapped the expression in `[]` to show that there are two matches returned. You do not have to wrap in `[]` in your path expression.", document: `{a: {name: frog, b: {name: blog, age: 12}}}`, expression: `[.. | select(has("name"))]`, expected: []string{ "D0, P[], (!!seq)::- {name: frog, b: {name: blog, age: 12}}\n- {name: blog, age: 12}\n", }, }, { description: "Recursively find nodes with values", document: `{a: {nameA: frog, b: {nameB: frog, age: 12}}}`, expression: `.. | select(. == "frog")`, expected: []string{ "D0, P[a nameA], (!!str)::frog\n", "D0, P[a b nameB], (!!str)::frog\n", }, }, { description: "Recurse map (values and keys)", subdescription: "Note that the map key appears in the results", document: `{a: frog}`, expression: `...`, expected: []string{ "D0, P[], (!!map)::{a: frog}\n", "D0, P[a], (!!str)::a\n", "D0, P[a], (!!str)::frog\n", }, }, { skipDoc: true, document: `{a: {b: apple}}`, expression: `..`, expected: []string{ "D0, P[], (!!map)::{a: {b: apple}}\n", "D0, P[a], (!!map)::{b: apple}\n", "D0, P[a b], (!!str)::apple\n", }, }, { skipDoc: true, document: `{a: {b: apple}}`, expression: `...`, expected: []string{ "D0, P[], (!!map)::{a: {b: apple}}\n", "D0, P[a], (!!str)::a\n", "D0, P[a], (!!map)::{b: apple}\n", "D0, P[a b], (!!str)::b\n", "D0, P[a b], (!!str)::apple\n", }, }, { skipDoc: true, document: `[1,2,3]`, expression: `..`, expected: []string{ "D0, P[], (!!seq)::[1, 2, 3]\n", "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!int)::3\n", }, }, { skipDoc: true, document: `[1,2,3]`, expression: `...`, expected: []string{ "D0, P[], (!!seq)::[1, 2, 3]\n", "D0, P[0], (!!int)::1\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!int)::3\n", }, }, { skipDoc: true, document: `[{a: cat},2,true]`, expression: `..`, expected: []string{ "D0, P[], (!!seq)::[{a: cat}, 2, true]\n", "D0, P[0], (!!map)::{a: cat}\n", "D0, P[0 a], (!!str)::cat\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!bool)::true\n", }, }, { skipDoc: true, document: `[{a: cat},2,true]`, expression: `...`, expected: []string{ "D0, P[], (!!seq)::[{a: cat}, 2, true]\n", "D0, P[0], (!!map)::{a: cat}\n", "D0, P[0 a], (!!str)::a\n", "D0, P[0 a], (!!str)::cat\n", "D0, P[1], (!!int)::2\n", "D0, P[2], (!!bool)::true\n", }, }, { description: "Aliases are not traversed", document: `{a: &cat {c: frog}, b: *cat}`, expression: `[..]`, expected: []string{ "D0, P[], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n", }, }, { skipDoc: true, document: `{a: &cat {c: frog}, b: *cat}`, expression: `...`, expected: []string{ "D0, P[], (!!map)::{a: &cat {c: frog}, b: *cat}\n", "D0, P[a], (!!str)::a\n", "D0, P[a], (!!map)::&cat {c: frog}\n", "D0, P[a c], (!!str)::c\n", "D0, P[a c], (!!str)::frog\n", "D0, P[b], (!!str)::b\n", "D0, P[b], (alias)::*cat\n", }, }, { description: "Merge docs are not traversed", document: mergeDocSample, expression: `.foobar | [..]`, expected: []string{ "D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobar | [...]`, expected: []string{ "D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList | ..`, expected: []string{ "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", "D0, P[foobarList b], (!!str)::foobarList_b\n", "D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n", "D0, P[foobarList << 0], (alias)::*foo\n", "D0, P[foobarList << 1], (alias)::*bar\n", "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList | ...`, expected: []string{ "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", "D0, P[foobarList b], (!!str)::b\n", "D0, P[foobarList b], (!!str)::foobarList_b\n", "D0, P[foobarList <<], (!!merge)::<<\n", "D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n", "D0, P[foobarList << 0], (alias)::*foo\n", "D0, P[foobarList << 1], (alias)::*bar\n", "D0, P[foobarList c], (!!str)::c\n", "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, } func TestRecursiveDescentOperatorScenarios(t *testing.T) { for _, tt := range recursiveDescentOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "recursive-descent-glob", recursiveDescentOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_reduce.go ================================================ package yqlib import ( "container/list" "fmt" ) func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("reduceOp") //.a as $var reduce (0; . + $var) //lhs is the assignment operator //rhs is the reduce block // '.' refers to the current accumulator, initialised to 0 // $var references a single element from the .a //ensure lhs is actually an assignment //and rhs is a block (empty) if expressionNode.LHS.Operation.OperationType != assignVariableOpType { return Context{}, fmt.Errorf("reduce must be given a variables assignment, got %v instead", expressionNode.LHS.Operation.OperationType.Type) } else if expressionNode.RHS.Operation.OperationType != blockOpType { return Context{}, fmt.Errorf("reduce must be given a block, got %v instead", expressionNode.RHS.Operation.OperationType.Type) } arrayExpNode := expressionNode.LHS.LHS array, err := d.GetMatchingNodes(context, arrayExpNode) if err != nil { return Context{}, err } variableName := expressionNode.LHS.RHS.Operation.StringValue initExp := expressionNode.RHS.LHS accum, err := d.GetMatchingNodes(context, initExp) if err != nil { return Context{}, err } log.Debugf("with variable %v", variableName) blockExp := expressionNode.RHS.RHS for el := array.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("REDUCING WITH %v", NodeToString(candidate)) l := list.New() l.PushBack(candidate) accum.SetVariable(variableName, l) accum, err = d.GetMatchingNodes(accum, blockExp) if err != nil { return Context{}, err } } return accum, nil } ================================================ FILE: pkg/yqlib/operator_reduce_test.go ================================================ package yqlib import ( "testing" ) var reduceOperatorScenarios = []expressionScenario{ { description: "Sum numbers", document: `[10,2, 5, 3]`, expression: `.[] as $item ireduce (0; . + $item)`, expected: []string{ "D0, P[], (!!int)::20\n", }, }, { description: "Merge all yaml files together", document: `a: cat`, document2: `b: dog`, expression: `. as $item ireduce ({}; . * $item )`, expected: []string{ "D0, P[], (!!map)::a: cat\nb: dog\n", }, }, { description: "Convert an array to an object", document: `[{name: Cathy, has: apples},{name: Bob, has: bananas}]`, expression: `.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )`, expected: []string{ "D0, P[], (!!map)::Cathy: apples\nBob: bananas\n", }, }, } func TestReduceOperatorScenarios(t *testing.T) { for _, tt := range reduceOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "reduce", reduceOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_reverse.go ================================================ package yqlib import ( "container/list" "fmt" ) func reverseOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.Tag) } reverseList := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style) reverseContent := make([]*CandidateNode, len(candidate.Content)) for i, originalNode := range candidate.Content { reverseContent[len(candidate.Content)-i-1] = originalNode } reverseList.AddChildren(reverseContent) results.PushBack(reverseList) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_reverse_test.go ================================================ package yqlib import "testing" var reverseOperatorScenarios = []expressionScenario{ { description: "Reverse", document: "[1, 2, 3]", expression: `reverse`, expected: []string{ "D0, P[], (!!seq)::[3, 2, 1]\n", }, }, { description: "Reverse", skipDoc: true, document: "[1, 2]", expression: `reverse[]`, expected: []string{ "D0, P[0], (!!int)::2\n", "D0, P[1], (!!int)::1\n", }, }, { skipDoc: true, document: "[]", expression: `reverse`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: "[1]", expression: `reverse`, expected: []string{ "D0, P[], (!!seq)::[1]\n", }, }, { skipDoc: true, document: "[1,2]", expression: `reverse`, expected: []string{ "D0, P[], (!!seq)::[2, 1]\n", }, }, { description: "Sort descending by string field", subdescription: "Use sort with reverse to sort in descending order.", document: "[{a: banana},{a: cat},{a: apple}]", expression: `sort_by(.a) | reverse`, expected: []string{ "D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n", }, }, { description: "Sort descending by string field, with comments", skipDoc: true, document: "# abc\n[{a: banana},{a: cat},{a: apple}]\n# xyz", expression: `sort_by(.a) | reverse`, expected: []string{ "D0, P[], (!!seq)::# abc\n[{a: cat}, {a: banana}, {a: apple}]\n# xyz\n", }, }, } func TestReverseOperatorScenarios(t *testing.T) { for _, tt := range reverseOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "reverse", reverseOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_select.go ================================================ package yqlib import ( "container/list" ) func selectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("selectOperation") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } // find any truthy node includeResult := false for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() { result := resultEl.Value.(*CandidateNode) includeResult = isTruthyNode(result) if includeResult { break } } if includeResult { results.PushBack(candidate) } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_select_test.go ================================================ package yqlib import ( "testing" ) var selectOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `cat: pants`, expression: `select(.nope) | key + " why though?"`, expected: []string{}, }, { skipDoc: true, document: `cat`, expression: `select(false, true)`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { skipDoc: true, document: `cat`, expression: `select(true, false)`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { skipDoc: true, document: `cat`, expression: `select(false)`, expected: []string{}, }, { description: "Select elements from array using wildcard prefix", document: `[cat,goat,dog]`, expression: `.[] | select(. == "*at")`, expected: []string{ "D0, P[0], (!!str)::cat\n", "D0, P[1], (!!str)::goat\n", }, }, { description: "Select elements from array using wildcard suffix", document: `[go-kart,goat,dog]`, expression: `.[] | select(. == "go*")`, expected: []string{ "D0, P[0], (!!str)::go-kart\n", "D0, P[1], (!!str)::goat\n", }, }, { description: "Select elements from array using wildcard prefix and suffix", document: `[ago, go, meow, going]`, expression: `.[] | select(. == "*go*")`, expected: []string{ "D0, P[0], (!!str)::ago\n", "D0, P[1], (!!str)::go\n", "D0, P[3], (!!str)::going\n", }, }, { description: "Select elements from array with regular expression", subdescription: "See more regular expression examples under the [`string` operator docs](https://mikefarah.gitbook.io/yq/operators/string-operators).", document: `[this_0, not_this, nor_0_this, thisTo_4]`, expression: `.[] | select(test("[a-zA-Z]+_[0-9]$"))`, expected: []string{ "D0, P[0], (!!str)::this_0\n", "D0, P[3], (!!str)::thisTo_4\n", }, }, { skipDoc: true, document: "a: hello", document2: "b: world", expression: `select(.a == "hello" or .b == "world")`, expected: []string{ "D0, P[], (!!map)::a: hello\n", "D0, P[], (!!map)::b: world\n", }, }, { description: "select splat", skipDoc: true, document: "a: hello", document2: "b: world", expression: `select(.a == "hello" or .b == "world")[]`, expected: []string{ "D0, P[a], (!!str)::hello\n", "D0, P[b], (!!str)::world\n", }, }, { description: "select does not update the map", skipDoc: true, document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`, expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`, expected: []string{ "D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n", }, }, { skipDoc: true, document: `[hot, fot, dog]`, expression: `.[] | select(. == "*at")`, expected: []string{}, }, { skipDoc: true, document: `a: [cat,goat,dog]`, expression: `.a.[] | select(. == "*at")`, expected: []string{ "D0, P[a 0], (!!str)::cat\n", "D0, P[a 1], (!!str)::goat\n"}, }, { description: "Select items from a map", document: `{ things: cat, bob: goat, horse: dog }`, expression: `.[] | select(. == "cat" or test("og$"))`, expected: []string{ "D0, P[things], (!!str)::cat\n", "D0, P[horse], (!!str)::dog\n", }, }, { description: "Use select and with_entries to filter map keys", document: `{name: bob, legs: 2, game: poker}`, expression: `with_entries(select(.key | test("ame$")))`, expected: []string{ "D0, P[], (!!map)::name: bob\ngame: poker\n", }, }, { description: "Select multiple items in a map and update", subdescription: "Note the brackets around the entire LHS.", document: `a: { things: cat, bob: goat, horse: dog }`, expression: `(.a.[] | select(. == "cat" or . == "goat")) |= "rabbit"`, expected: []string{ "D0, P[], (!!map)::a: {things: rabbit, bob: rabbit, horse: dog}\n", }, }, { skipDoc: true, document: `a: { things: {include: true}, notMe: {include: false}, andMe: {include: fold} }`, expression: `.a.[] | select(.include)`, expected: []string{ "D0, P[a things], (!!map)::{include: true}\n", "D0, P[a andMe], (!!map)::{include: fold}\n", }, }, { skipDoc: true, document: `[cat,~,dog]`, expression: `.[] | select(. == ~)`, expected: []string{ "D0, P[1], (!!null)::~\n", }, }, } func TestSelectOperatorScenarios(t *testing.T) { for _, tt := range selectOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "select", selectOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_self.go ================================================ package yqlib func selfOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { return context, nil } ================================================ FILE: pkg/yqlib/operator_shuffle.go ================================================ package yqlib import ( "container/list" "fmt" "math/rand" ) func shuffleOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { // ignore CWE-338 gosec issue of not using crypto/rand // this is just to shuffle an array rather generating a // secret or something that needs proper rand. myRand := rand.New(rand.NewSource(Now().UnixNano())) // #nosec results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.Tag) } result := candidate.Copy() a := result.Content myRand.Shuffle(len(a), func(i, j int) { a[i], a[j] = a[j], a[i] oldIndex := a[i].Key.Value a[i].Key.Value = a[j].Key.Value a[j].Key.Value = oldIndex }) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_shuffle_test.go ================================================ package yqlib import "testing" var shuffleOperatorScenarios = []expressionScenario{ { description: "Shuffle array", document: "[1, 2, 3, 4, 5]", expression: `shuffle`, expected: []string{ "D0, P[], (!!seq)::[5, 2, 4, 1, 3]\n", }, }, { description: "Shuffle array", skipDoc: true, document: "[1, 2, 3]", expression: `shuffle[]`, expected: []string{ "D0, P[0], (!!int)::3\n", "D0, P[1], (!!int)::1\n", "D0, P[2], (!!int)::2\n", }, }, { description: "Shuffle array in place", document: "cool: [1, 2, 3, 4, 5]", expression: `.cool |= shuffle`, expected: []string{ "D0, P[], (!!map)::cool: [5, 2, 4, 1, 3]\n", }, }, } func TestShuffleByOperatorScenarios(t *testing.T) { for _, tt := range shuffleOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "shuffle", shuffleOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_slice.go ================================================ package yqlib import ( "container/list" "fmt" ) func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode, expressionNode *ExpressionNode) (int, error) { result, err := d.GetMatchingNodes(context.SingleChildContext(node), expressionNode) if err != nil { return 0, err } if result.MatchingNodes.Len() != 1 { return 0, fmt.Errorf("expected to find 1 number, got %v instead", result.MatchingNodes.Len()) } return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value) } func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("slice array operator!") log.Debug("lhs: %v", expressionNode.LHS.Operation.toString()) log.Debug("rhs: %v", expressionNode.RHS.Operation.toString()) results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { lhsNode := el.Value.(*CandidateNode) firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS) if err != nil { return Context{}, err } relativeFirstNumber := firstNumber if relativeFirstNumber < 0 { relativeFirstNumber = len(lhsNode.Content) + firstNumber } secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS) if err != nil { return Context{}, err } relativeSecondNumber := secondNumber if relativeSecondNumber < 0 { relativeSecondNumber = len(lhsNode.Content) + secondNumber } else if relativeSecondNumber > len(lhsNode.Content) { relativeSecondNumber = len(lhsNode.Content) } log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber) var newResults []*CandidateNode for i := relativeFirstNumber; i < relativeSecondNumber; i++ { newResults = append(newResults, lhsNode.Content[i]) } sliceArrayNode := lhsNode.CreateReplacement(SequenceNode, lhsNode.Tag, "") sliceArrayNode.AddChildren(newResults) results.PushBack(sliceArrayNode) } // result is now the context that has the nodes we need to put back into a sequence. //what about multiple arrays in the context? I think we need to create an array for each one return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_slice_test.go ================================================ package yqlib import "testing" var sliceArrayScenarios = []expressionScenario{ { description: "Slicing arrays", document: `[cat, dog, frog, cow]`, expression: `.[1:3]`, expected: []string{ "D0, P[], (!!seq)::- dog\n- frog\n", }, }, { description: "Slicing arrays - without the first number", subdescription: "Starts from the start of the array", document: `[cat, dog, frog, cow]`, expression: `.[:2]`, expected: []string{ "D0, P[], (!!seq)::- cat\n- dog\n", }, }, { description: "Slicing arrays - without the second number", subdescription: "Finishes at the end of the array", document: `[cat, dog, frog, cow]`, expression: `.[2:]`, expected: []string{ "D0, P[], (!!seq)::- frog\n- cow\n", }, }, { description: "Slicing arrays - use negative numbers to count backwards from the end", document: `[cat, dog, frog, cow]`, expression: `.[1:-1]`, expected: []string{ "D0, P[], (!!seq)::- dog\n- frog\n", }, }, { description: "Inserting into the middle of an array", subdescription: "using an expression to find the index", document: `[cat, dog, frog, cow]`, expression: `(.[] | select(. == "dog") | key + 1) as $pos | .[0:($pos)] + ["rabbit"] + .[$pos:]`, expected: []string{ "D0, P[], (!!seq)::- cat\n- dog\n- rabbit\n- frog\n- cow\n", }, }, { skipDoc: true, document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`, expression: `.[] | .[1:3]`, expected: []string{ "D0, P[0], (!!seq)::- dog\n- frog\n", "D0, P[1], (!!seq)::- banana\n- grape\n", }, }, { skipDoc: true, description: "second index beyond array clamps", document: `[cat]`, expression: `.[:3]`, expected: []string{ "D0, P[], (!!seq)::- cat\n", }, }, { skipDoc: true, description: "first index beyond array returns nothing", document: `[cat]`, expression: `.[3:]`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`, expression: `.[] | .[-2:-1]`, expected: []string{ "D0, P[0], (!!seq)::- frog\n", "D0, P[1], (!!seq)::- grape\n", }, }, { skipDoc: true, document: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`, expression: `.[10:11]`, expected: []string{ "D0, P[], (!!seq)::- cat11\n", }, }, { skipDoc: true, document: `[cat1, cat2, cat3, cat4, cat5, cat6, cat7, cat8, cat9, cat10, cat11]`, expression: `.[-11:-10]`, expected: []string{ "D0, P[], (!!seq)::- cat1\n", }, }, } func TestSliceOperatorScenarios(t *testing.T) { for _, tt := range sliceArrayScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "slice-array", sliceArrayScenarios) } ================================================ FILE: pkg/yqlib/operator_sort.go ================================================ package yqlib import ( "container/list" "fmt" "sort" "strconv" "strings" "time" ) func sortOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} expressionNode.RHS = selfExpression return sortByOperator(d, context, expressionNode) } // context represents the current matching nodes in the expression pipeline // expressionNode is your current expression (sort_by) func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) var sortableArray sortableNodeArray if candidate.CanVisitValues() { sortableArray = make(sortableNodeArray, 0) visitor := func(valueNode *CandidateNode) error { compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(valueNode), expressionNode.RHS) if err != nil { return err } sortableNode := sortableNode{Node: valueNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()} sortableArray = append(sortableArray, sortableNode) return nil } if err := candidate.VisitValues(visitor); err != nil { return context, err } } else { return context, fmt.Errorf("node at path [%v] is not an array or map (it's a %v)", candidate.GetNicePath(), candidate.Tag) } sort.Stable(sortableArray) sortedList := candidate.CopyWithoutContent() switch candidate.Kind { case MappingNode: for _, sortedNode := range sortableArray { sortedList.AddKeyValueChild(sortedNode.Node.Key, sortedNode.Node) } case SequenceNode: for _, sortedNode := range sortableArray { sortedList.AddChild(sortedNode.Node) } } // convert array of value nodes back to map results.PushBack(sortedList) } return context.ChildContext(results), nil } type sortableNode struct { Node *CandidateNode CompareContext Context dateTimeLayout string } type sortableNodeArray []sortableNode func (a sortableNodeArray) Len() int { return len(a) } func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a sortableNodeArray) Less(i, j int) bool { lhsContext := a[i].CompareContext rhsContext := a[j].CompareContext rhsEl := rhsContext.MatchingNodes.Front() for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() { lhs := lhsEl.Value.(*CandidateNode) rhs := rhsEl.Value.(*CandidateNode) result := a.compare(lhs, rhs, a[i].dateTimeLayout) if result < 0 { return true } else if result > 0 { return false } rhsEl = rhsEl.Next() } return lhsContext.MatchingNodes.Len() < rhsContext.MatchingNodes.Len() } func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateTimeLayout string) int { lhsTag := lhs.Tag rhsTag := rhs.Tag if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() } if !strings.HasPrefix(rhsTag, "!!") { // custom tag - we have to have a guess rhsTag = rhs.guessTagFromCustomType() } isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp" layout := dateTimeLayout // if the lhs is a string, it might be a timestamp in a custom format. if lhsTag == "!!str" && layout != time.RFC3339 { _, errLhs := parseDateTime(layout, lhs.Value) _, errRhs := parseDateTime(layout, rhs.Value) isDateTime = errLhs == nil && errRhs == nil } if lhsTag == "!!null" && rhsTag != "!!null" { return -1 } else if lhsTag != "!!null" && rhsTag == "!!null" { return 1 } else if lhsTag == "!!bool" && rhsTag != "!!bool" { return -1 } else if lhsTag != "!!bool" && rhsTag == "!!bool" { return 1 } else if lhsTag == "!!bool" && rhsTag == "!!bool" { lhsTruthy := isTruthyNode(lhs) rhsTruthy := isTruthyNode(rhs) if lhsTruthy == rhsTruthy { return 0 } else if lhsTruthy { return 1 } return -1 } else if isDateTime { lhsTime, err := parseDateTime(layout, lhs.Value) if err != nil { log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err) return strings.Compare(lhs.Value, rhs.Value) } rhsTime, err := parseDateTime(layout, rhs.Value) if err != nil { log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err) return strings.Compare(lhs.Value, rhs.Value) } if lhsTime.Equal(rhsTime) { return 0 } else if lhsTime.Before(rhsTime) { return -1 } return 1 } else if lhsTag == "!!int" && rhsTag == "!!int" { _, lhsNum, err := parseInt64(lhs.Value) if err != nil { panic(err) } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { panic(err) } return int(lhsNum - rhsNum) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { panic(err) } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { panic(err) } if lhsNum == rhsNum { return 0 } else if lhsNum < rhsNum { return -1 } return 1 } return strings.Compare(lhs.Value, rhs.Value) } ================================================ FILE: pkg/yqlib/operator_sort_keys.go ================================================ package yqlib import ( "sort" ) func sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } for childEl := rhs.MatchingNodes.Front(); childEl != nil; childEl = childEl.Next() { node := childEl.Value.(*CandidateNode) if node.Kind == MappingNode { sortKeys(node) } if err != nil { return Context{}, err } } } return context, nil } func sortKeys(node *CandidateNode) { keys := make([]string, len(node.Content)/2) keyBucket := map[string]*CandidateNode{} valueBucket := map[string]*CandidateNode{} var contents = node.Content for index := 0; index < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] keys[index/2] = key.Value keyBucket[key.Value] = key valueBucket[key.Value] = value } sort.Strings(keys) sortedContent := make([]*CandidateNode, len(node.Content)) for index := 0; index < len(keys); index = index + 1 { keyString := keys[index] sortedContent[index*2] = keyBucket[keyString] sortedContent[1+(index*2)] = valueBucket[keyString] } // re-arranging children, no need to update their parent // relationship node.Content = sortedContent } ================================================ FILE: pkg/yqlib/operator_sort_keys_test.go ================================================ package yqlib import ( "testing" ) var sortKeysOperatorScenarios = []expressionScenario{ { description: "Sort keys of map", document: `{c: frog, a: blah, b: bing}`, expression: `sort_keys(.)`, expected: []string{ "D0, P[], (!!map)::{a: blah, b: bing, c: frog}\n", }, }, { description: "Sort keys of map", skipDoc: true, document: `{c: frog, a: zoo}`, expression: `sort_keys(.)[]`, expected: []string{ "D0, P[a], (!!str)::zoo\n", "D0, P[c], (!!str)::frog\n", }, }, { skipDoc: true, document: `{c: frog}`, expression: `sort_keys(.d)`, expected: []string{ "D0, P[], (!!map)::{c: frog}\n", }, }, { description: "Sort keys recursively", subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted", document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`, expression: `sort_keys(..)`, expected: []string{ "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", }, }, } func TestSortKeysOperatorScenarios(t *testing.T) { for _, tt := range sortKeysOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "sort-keys", sortKeysOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_sort_test.go ================================================ package yqlib import "testing" var sortByOperatorScenarios = []expressionScenario{ { description: "Sort by string field", document: "[{a: banana},{a: cat},{a: apple}]", expression: `sort_by(.a)`, expected: []string{ "D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n", }, }, { description: "Sort by string field", skipDoc: true, document: "[{a: banana},{a: apple}]", expression: `sort_by(.a)[]`, expected: []string{ "D0, P[0], (!!map)::{a: apple}\n", "D0, P[1], (!!map)::{a: banana}\n", }, }, { description: "Sort by with null", skipDoc: true, document: "[{a: banana},null,{a: apple}]", expression: `sort_by(.a)[]`, expected: []string{ "D0, P[0], (!!null)::null\n", "D0, P[1], (!!map)::{a: apple}\n", "D0, P[2], (!!map)::{a: banana}\n", }, }, { description: "Sort by multiple fields", document: "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]", expression: `sort_by(.a, .b)`, expected: []string{ "D0, P[], (!!seq)::[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}]\n", }, }, { description: "Sort by multiple fields", skipDoc: true, document: "[{a: dog, b: good},{a: cat, c: things},{a: cat, b: apple}]", expression: `sort_by(.a, .b)`, expected: []string{ "D0, P[], (!!seq)::[{a: cat, c: things}, {a: cat, b: apple}, {a: dog, b: good}]\n", }, }, { description: "Sort by multiple fields", skipDoc: true, document: "[{a: dog, b: 0.1},{a: cat, b: 0.01},{a: cat, b: 0.001}]", expression: `sort_by(.a, .b)`, expected: []string{ "D0, P[], (!!seq)::[{a: cat, b: 0.001}, {a: cat, b: 0.01}, {a: dog, b: 0.1}]\n", }, }, { description: "Sort descending by string field", subdescription: "Use sort with reverse to sort in descending order.", document: "[{a: banana},{a: cat},{a: apple}]", expression: `sort_by(.a) | reverse`, expected: []string{ "D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n", }, }, { description: "Sort array in place", document: "cool: [{a: banana},{a: cat},{a: apple}]", expression: `.cool |= sort_by(.a)`, expected: []string{ "D0, P[], (!!map)::cool: [{a: apple}, {a: banana}, {a: cat}]\n", }, }, { description: "Sort array of objects by key", subdescription: "Note that you can give sort_by complex expressions, not just paths", document: "cool: [{b: banana},{a: banana},{c: banana}]", expression: `.cool |= sort_by(keys | .[0])`, expected: []string{ "D0, P[], (!!map)::cool: [{a: banana}, {b: banana}, {c: banana}]\n", }, }, { description: "Sort a map", subdescription: "Sorting a map, by default this will sort by the values", document: "y: b\nz: a\nx: c\n", expression: `sort`, expected: []string{ "D0, P[], (!!map)::z: a\ny: b\nx: c\n", }, }, { description: "Sort a map by keys", subdescription: "Use sort_by to sort a map using a custom function", document: "Y: b\nz: a\nx: c\n", expression: `sort_by(key | downcase)`, expected: []string{ "D0, P[], (!!map)::x: c\nY: b\nz: a\n", }, }, { description: "Sort is stable", subdescription: "Note the order of the elements in unchanged when equal in sorting.", document: "[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]", expression: `sort_by(.a)`, expected: []string{ "D0, P[], (!!seq)::[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\n", }, }, { description: "Sort by numeric field", document: "[{a: 10},{a: 100},{a: 1}]", expression: `sort_by(.a)`, expected: []string{ "D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n", }, }, { description: "Sort by custom date field", document: `[{a: "12-Jun-2011"},{a: "23-Dec-2010"},{a: "10-Aug-2011"}]`, expression: `with_dtf("02-Jan-2006"; sort_by(.a))`, expected: []string{ "D0, P[], (!!seq)::[{a: \"23-Dec-2010\"}, {a: \"12-Jun-2011\"}, {a: \"10-Aug-2011\"}]\n", }, }, { skipDoc: true, document: "[{a: 1.1},{a: 1.001},{a: 1.01}]", expression: `sort_by(.a)`, expected: []string{ "D0, P[], (!!seq)::[{a: 1.001}, {a: 1.01}, {a: 1.1}]\n", }, }, { description: "Sort, nulls come first", document: "[8,3,null,6, true, false, cat]", expression: `sort`, expected: []string{ "D0, P[], (!!seq)::[null, false, true, 3, 6, 8, cat]\n", }, }, { description: "Sort, nulls come first", skipDoc: true, document: "[8,null]", expression: `sort[]`, expected: []string{ "D0, P[0], (!!null)::null\n", "D0, P[1], (!!int)::8\n", }, }, { skipDoc: true, description: "false before true", document: "[{a: false, b: 1}, {a: true, b: 2}, {a: false, b: 3}]", expression: `sort_by(.a)`, expected: []string{ "D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\n", }, }, { skipDoc: true, description: "head comment", document: "# abc\n- def\n# ghi", expression: `sort`, expected: []string{ "D0, P[], (!!seq)::# abc\n- def\n# ghi\n", }, }, } func TestSortByOperatorScenarios(t *testing.T) { for _, tt := range sortByOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "sort", sortByOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_split_document.go ================================================ package yqlib func splitDocumentOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("splitDocumentOperator") var index uint for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) candidate.SetDocument(index) candidate.SetParent(nil) index = index + 1 } return context, nil } ================================================ FILE: pkg/yqlib/operator_split_document_test.go ================================================ package yqlib import ( "testing" ) var splitDocOperatorScenarios = []expressionScenario{ { description: "Split empty", document: ``, expression: `split_doc`, expected: []string{ "D0, P[], (!!null)::\n", }, }, { description: "Split array", document: `[{a: cat}, {b: dog}]`, expression: `.[] | split_doc`, expected: []string{ "D0, P[0], (!!map)::{a: cat}\n", "D1, P[1], (!!map)::{b: dog}\n", }, }, { description: "Split splat", skipDoc: true, document: `[{a: cat}, {b: dog}]`, expression: `.[] | split_doc[]`, expected: []string{ "D0, P[0 a], (!!str)::cat\n", "D1, P[1 b], (!!str)::dog\n", }, }, } func TestSplitDocOperatorScenarios(t *testing.T) { for _, tt := range splitDocOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "split-into-documents", splitDocOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_strings.go ================================================ package yqlib import ( "container/list" "fmt" "regexp" "strings" ) var StringInterpolationEnabled = true type changeCasePrefs struct { ToUpperCase bool } func encodeToYamlString(node *CandidateNode) (string, error) { encoderPrefs := encoderPreferences{ format: YamlFormat, indent: ConfiguredYamlPreferences.Indent, } result, err := encodeToString(node, encoderPrefs) if err != nil { return "", err } return chomper.ReplaceAllString(result, ""), nil } func evaluate(d *dataTreeNavigator, context Context, expStr string) (string, error) { exp, err := ExpressionParser.ParseExpression(expStr) if err != nil { return "", err } result, err := d.GetMatchingNodes(context, exp) if err != nil { return "", err } if result.MatchingNodes.Len() == 0 { return "", nil } node := result.MatchingNodes.Front().Value.(*CandidateNode) if node.Kind != ScalarNode { return encodeToYamlString(node) } return node.Value, nil } func interpolate(d *dataTreeNavigator, context Context, str string) (string, error) { var sb strings.Builder var expSb strings.Builder inExpression := false nestedBracketsCounter := 0 runes := []rune(str) for i := 0; i < len(runes); i++ { char := runes[i] if !inExpression { if char == '\\' && i < len(runes)-1 { switch runes[i+1] { case '(': inExpression = true // skip the lparen i++ continue case '\\': // skip the escaped backslash i++ default: log.Debugf("Ignoring non-escaping backslash @ %v[%d]", str, i) } } sb.WriteRune(char) } else { // we are in an expression if char == ')' { if nestedBracketsCounter == 0 { // finished the expression! log.Debugf("Expression is :%v", expSb.String()) value, err := evaluate(d, context, expSb.String()) if err != nil { return "", err } inExpression = false expSb = strings.Builder{} // reset this sb.WriteString(value) continue } nestedBracketsCounter-- } else if char == '(' { nestedBracketsCounter++ } else if char == '\\' && i < len(runes)-1 { switch esc := runes[i+1]; esc { case ')', '\\': // write escaped character expSb.WriteRune(esc) i++ continue default: log.Debugf("Ignoring non-escaping backslash @ %v[%d]", str, i) } } expSb.WriteRune(char) } } if inExpression { log.Warning("unclosed interpolation string, skipping interpolation") return str, nil } return sb.String(), nil } func stringInterpolationOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !StringInterpolationEnabled { return context.SingleChildContext( createScalarNode(expressionNode.Operation.StringValue, expressionNode.Operation.StringValue), ), nil } if context.MatchingNodes.Len() == 0 { value, err := interpolate(d, context, expressionNode.Operation.StringValue) if err != nil { return Context{}, err } node := createScalarNode(value, value) return context.SingleChildContext(node), nil } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) value, err := interpolate(d, context.SingleChildContext(candidate), expressionNode.Operation.StringValue) if err != nil { return Context{}, err } node := createScalarNode(value, value) results.PushBack(node) } return context.ChildContext(results), nil } func trimSpaceOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot trim %v, can only operate on strings. ", node.Tag) } newStringNode := node.CreateReplacement(ScalarNode, node.Tag, strings.TrimSpace(node.Value)) newStringNode.Style = node.Style results.PushBack(newStringNode) } return context.ChildContext(results), nil } func toStringOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { results := list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) var newStringNode *CandidateNode if node.Tag == "!!str" { newStringNode = node.CreateReplacement(ScalarNode, "!!str", node.Value) } else if node.Kind == ScalarNode { newStringNode = node.CreateReplacement(ScalarNode, "!!str", node.Value) newStringNode.Style = DoubleQuotedStyle } else { result, err := encodeToYamlString(node) if err != nil { return Context{}, err } newStringNode = node.CreateReplacement(ScalarNode, "!!str", result) newStringNode.Style = DoubleQuotedStyle } newStringNode.Tag = "!!str" results.PushBack(newStringNode) } return context.ChildContext(results), nil } func changeCaseOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { results := list.New() prefs := expressionNode.Operation.Preferences.(changeCasePrefs) for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot change case with %v, can only operate on strings. ", node.Tag) } value := "" if prefs.ToUpperCase { value = strings.ToUpper(node.Value) } else { value = strings.ToLower(node.Value) } newStringNode := node.CreateReplacement(ScalarNode, node.Tag, value) newStringNode.Style = node.Style results.PushBack(newStringNode) } return context.ChildContext(results), nil } func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) { regEx := "" replacementText := "" regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) if err != nil { return "", "", err } if regExNodes.MatchingNodes.Front() != nil { regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } log.Debug("regEx %v", regEx) replacementNodes, err := d.GetMatchingNodes(context, block.RHS) if err != nil { return "", "", err } if replacementNodes.MatchingNodes.Front() != nil { replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } return regEx, replacementText, nil } func substitute(original string, regex *regexp.Regexp, replacement string) (Kind, string, string) { replacedString := regex.ReplaceAllString(original, replacement) return ScalarNode, "!!str", replacedString } func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { //rhs block operator //lhs of block = regex //rhs of block = replacement expression block := expressionNode.RHS regExStr, replacementText, err := getSubstituteParameters(d, block, context) if err != nil { return Context{}, err } regEx, err := regexp.Compile(regExStr) if err != nil { return Context{}, err } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } result := node.CreateReplacement(substitute(node.Value, regEx, replacementText)) results.PushBack(result) } return context.ChildContext(results), nil } func addMatch(original []*CandidateNode, match string, offset int, name string) []*CandidateNode { newContent := append(original, createScalarNode("string", "string")) if offset < 0 { // offset of -1 means there was no match, force a null value like jq newContent = append(newContent, createScalarNode(nil, "null"), ) } else { newContent = append(newContent, createScalarNode(match, match), ) } newContent = append(newContent, createScalarNode("offset", "offset"), createScalarNode(offset, fmt.Sprintf("%v", offset)), createScalarNode("length", "length"), createScalarNode(len(match), fmt.Sprintf("%v", len(match)))) if name != "" { newContent = append(newContent, createScalarNode("name", "name"), createScalarNode(name, name), ) } return newContent } type matchPreferences struct { Global bool } func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) { var allMatches [][]string var allIndices [][]int if matchPrefs.Global { allMatches = regEx.FindAllStringSubmatch(value, -1) allIndices = regEx.FindAllStringSubmatchIndex(value, -1) } else { allMatches = [][]string{regEx.FindStringSubmatch(value)} allIndices = [][]int{regEx.FindStringSubmatchIndex(value)} } log.Debug("allMatches, %v", allMatches) return allMatches, allIndices } func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) { subNames := regEx.SubexpNames() allMatches, allIndices := getMatches(matchPrefs, regEx, value) // if all matches just has an empty array in it, // then nothing matched if len(allMatches) > 0 && len(allMatches[0]) == 0 { return } for i, matches := range allMatches { capturesListNode := &CandidateNode{Kind: SequenceNode} match, submatches := matches[0], matches[1:] for j, submatch := range submatches { captureNode := &CandidateNode{Kind: MappingNode} captureNode.AddChildren(addMatch(captureNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1])) capturesListNode.AddChild(captureNode) } node := candidate.CreateReplacement(MappingNode, "!!map", "") node.AddChildren(addMatch(node.Content, match, allIndices[i][0], "")) node.AddKeyValueChild(createScalarNode("captures", "captures"), capturesListNode) results.PushBack(node) } } func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) { subNames := regEx.SubexpNames() allMatches, allIndices := getMatches(matchPrefs, regEx, value) // if all matches just has an empty array in it, // then nothing matched if len(allMatches) > 0 && len(allMatches[0]) == 0 { return } for i, matches := range allMatches { capturesNode := candidate.CreateReplacement(MappingNode, "!!map", "") _, submatches := matches[0], matches[1:] for j, submatch := range submatches { keyNode := createScalarNode(subNames[j+1], subNames[j+1]) var valueNode *CandidateNode offset := allIndices[i][2+j*2] // offset of -1 means there was no match, force a null value like jq if offset < 0 { valueNode = createScalarNode(nil, "null") } else { valueNode = createScalarNode(submatch, submatch) } capturesNode.AddKeyValueChild(keyNode, valueNode) } results.PushBack(capturesNode) } } func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) { regExExpNode := expressionNode.RHS matchPrefs := matchPreferences{} // we got given parameters e.g. match(exp; params) if expressionNode.RHS.Operation.OperationType == blockOpType { block := expressionNode.RHS regExExpNode = block.LHS replacementNodes, err := d.GetMatchingNodes(context, block.RHS) if err != nil { return nil, matchPrefs, err } paramText := "" if replacementNodes.MatchingNodes.Front() != nil { paramText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } if strings.Contains(paramText, "g") { paramText = strings.ReplaceAll(paramText, "g", "") matchPrefs.Global = true } if strings.Contains(paramText, "i") { return nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match("(?i)cat")`) } if len(paramText) > 0 { return nil, matchPrefs, fmt.Errorf(`unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText) } } regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode) if err != nil { return nil, matchPrefs, err } log.Debug(NodesToString(regExNodes.MatchingNodes)) regExStr := "" if regExNodes.MatchingNodes.Front() != nil { regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } log.Debug("regEx %v", regExStr) regEx, err := regexp.Compile(regExStr) return regEx, matchPrefs, err } func matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode) if err != nil { return Context{}, err } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } match(matchPrefs, regEx, node, node.Value, results) } return context.ChildContext(results), nil } func captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode) if err != nil { return Context{}, err } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } capture(matchPrefs, regEx, node, node.Value, results) } return context.ChildContext(results), nil } func testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { regEx, _, err := extractMatchArguments(d, context, expressionNode) if err != nil { return Context{}, err } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } matches := regEx.FindStringSubmatch(node.Value) results.PushBack(createBooleanCandidate(node, len(matches) > 0)) } return context.ChildContext(results), nil } func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("joinStringOperator") joinStr := "" rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { joinStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.Kind != SequenceNode { return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag) } result := node.CreateReplacement(join(node.Content, joinStr)) results.PushBack(result) } return context.ChildContext(results), nil } func join(content []*CandidateNode, joinStr string) (Kind, string, string) { var stringsToJoin []string for _, node := range content { str := node.Value if node.Tag == "!!null" { str = "" } stringsToJoin = append(stringsToJoin, str) } return ScalarNode, "!!str", strings.Join(stringsToJoin, joinStr) } func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("splitStringOperator") splitStr := "" rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { splitStr = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) if node.Tag == "!!null" { continue } if node.guessTagFromCustomType() != "!!str" { return Context{}, fmt.Errorf("cannot split %v, can only split strings", node.Tag) } kind, tag, content := split(node.Value, splitStr) result := node.CreateReplacement(kind, tag, "") result.AddChildren(content) results.PushBack(result) } return context.ChildContext(results), nil } func split(value string, spltStr string) (Kind, string, []*CandidateNode) { var contents []*CandidateNode if value != "" { log.Debug("going to spltStr[%v]", spltStr) var newStrings = strings.Split(value, spltStr) contents = make([]*CandidateNode, len(newStrings)) for index, str := range newStrings { contents[index] = &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: str} } } return SequenceNode, "!!seq", contents } ================================================ FILE: pkg/yqlib/operator_strings_test.go ================================================ package yqlib import ( "testing" ) var stringsOperatorScenarios = []expressionScenario{ { description: "Interpolation", document: "value: things\nanother: stuff", expression: `.message = "I like \(.value) and \(.another)"`, expected: []string{ "D0, P[], (!!map)::value: things\nanother: stuff\nmessage: I like things and stuff\n", }, }, { description: "Interpolation - not a string", document: `value: {an: apple}`, expression: `.message = "I like \(.value)"`, expected: []string{ "D0, P[], (!!map)::value: {an: apple}\nmessage: 'I like {an: apple}'\n", }, }, { skipDoc: true, description: "Interpolation - just escape", expression: `"\\"`, expected: []string{ "D0, P[], (!!str)::\\\n", }, }, { skipDoc: true, description: "Interpolation - nested", document: `value: things`, expression: `"Hi \( (.value) )"`, expected: []string{ "D0, P[], (!!str)::Hi things\n", }, }, { skipDoc: true, description: "Interpolation - don't", document: `value: things`, expression: `"Hi (.value)"`, expected: []string{ "D0, P[], (!!str)::Hi (.value)\n", }, }, { skipDoc: true, description: "Interpolation - don't!", document: `value: things`, expression: `"Hi \\(.value)"`, expected: []string{ "D0, P[], (!!str)::Hi \\(.value)\n", }, }, { skipDoc: true, description: "Interpolation - random close bracket", document: `value: things`, expression: `"Hi )"`, expected: []string{ "D0, P[], (!!str)::Hi )\n", }, }, { skipDoc: true, description: "Interpolation - unclosed interpolation string", document: `value: things`, expression: `"Hi \("`, expected: []string{ "D0, P[], (!!str)::Hi \\(\n", }, }, { skipDoc: true, description: "Interpolation - unclosed interpolation string due to escape", document: `value: things`, expression: `"Hi \(\)"`, expected: []string{ "D0, P[], (!!str)::Hi \\(\\)\n", }, }, { description: "To up (upper) case", subdescription: "Works with unicode characters", document: `água`, expression: "upcase", expected: []string{ "D0, P[], (!!str)::ÁGUA\n", }, }, { skipDoc: true, document: `!camel água`, expression: "upcase", expected: []string{ "D0, P[], (!camel)::ÁGUA\n", }, }, { description: "To down (lower) case", subdescription: "Works with unicode characters", document: `ÁgUA`, expression: "downcase", expected: []string{ "D0, P[], (!!str)::água\n", }, }, { skipDoc: true, document: `!camel ÁgUA`, expression: "downcase", expected: []string{ "D0, P[], (!camel)::água\n", }, }, { description: "Join strings", document: `[cat, meow, 1, null, true]`, expression: `join("; ")`, expected: []string{ "D0, P[], (!!str)::cat; meow; 1; ; true\n", }, }, { description: "Trim strings", document: `[" cat", "dog ", " cow cow ", horse]`, expression: `.[] | trim`, expected: []string{ "D0, P[0], (!!str)::cat\n", "D0, P[1], (!!str)::dog\n", "D0, P[2], (!!str)::cow cow\n", "D0, P[3], (!!str)::horse\n", }, }, { skipDoc: true, document: `[!horse cat, !goat meow, !frog 1, null, true]`, expression: `join("; ")`, expected: []string{ "D0, P[], (!!str)::cat; meow; 1; ; true\n", }, }, { description: "Match string", document: `foo bar foo`, expression: `match("foo")`, expected: []string{ "D0, P[], (!!map)::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { skipDoc: true, document: `!horse foo bar foo`, expression: `match("foo")`, expected: []string{ "D0, P[], (!!map)::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { description: "Match string, case insensitive", document: `foo bar FOO`, expression: `[match("(?i)foo"; "g")]`, expected: []string{ "D0, P[], (!!seq)::- string: foo\n offset: 0\n length: 3\n captures: []\n- string: FOO\n offset: 8\n length: 3\n captures: []\n", }, }, { description: "Match with global capture group", document: `abc abc`, expression: `[match("(ab)(c)"; "g")]`, expected: []string{ "D0, P[], (!!seq)::- string: abc\n offset: 0\n length: 3\n captures:\n - string: ab\n offset: 0\n length: 2\n - string: c\n offset: 2\n length: 1\n- string: abc\n offset: 4\n length: 3\n captures:\n - string: ab\n offset: 4\n length: 2\n - string: c\n offset: 6\n length: 1\n", }, }, { description: "Match with named capture groups", document: `foo bar foo foo foo`, expression: `[match("foo (?Pbar)? foo"; "g")]`, expected: []string{ "D0, P[], (!!seq)::- string: foo bar foo\n offset: 0\n length: 11\n captures:\n - string: bar\n offset: 4\n length: 3\n name: bar123\n- string: foo foo\n offset: 12\n length: 8\n captures:\n - string: null\n offset: -1\n length: 0\n name: bar123\n", }, }, { description: "Capture named groups into a map", document: `xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, expected: []string{ "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\n", }, }, { skipDoc: true, document: `!horse xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, expected: []string{ "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\n", }, }, { skipDoc: true, description: "Capture named groups into a map, with null", document: `xyzzy-14`, expression: `capture("(?P[a-z]+)-(?P[0-9]+)(?Pbar)?")`, expected: []string{ "D0, P[], (!!map)::a: xyzzy\nn: \"14\"\nbar123: null\n", }, }, { description: "Match without global flag", document: `cat cat`, expression: `match("cat")`, expected: []string{ "D0, P[], (!!map)::string: cat\noffset: 0\nlength: 3\ncaptures: []\n", }, }, { description: "Match with global flag", document: `cat cat`, expression: `[match("cat"; "g")]`, expected: []string{ "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", }, }, { skipDoc: true, document: `!horse cat cat`, expression: `[match("cat"; "g")]`, expected: []string{ "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", }, }, { skipDoc: true, description: "No match", document: `dog`, expression: `match("cat"; "g")`, expected: []string{}, }, { skipDoc: true, description: "No match", expression: `"dog" | match("cat", "g")`, expected: []string{}, }, { skipDoc: true, description: "No match", expression: `"dog" | match("cat")`, expected: []string{}, }, { description: "Test using regex", subdescription: "Like jq's equivalent, this works like match but only returns true/false instead of full match details", document: `["cat", "dog"]`, expression: `.[] | test("at")`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::false\n", }, }, { skipDoc: true, document: `[!horse "cat", !cat "dog"]`, expression: `.[] | test("at")`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::false\n", }, }, { skipDoc: true, document: `["cat*", "cat*", "cat"]`, expression: `.[] | test("cat\*")`, expected: []string{ "D0, P[0], (!!bool)::true\n", "D0, P[1], (!!bool)::true\n", "D0, P[2], (!!bool)::false\n", }, }, { description: "Substitute / Replace string", subdescription: "This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\nNote the use of `|=` to run in context of the current string value.", document: `a: dogs are great`, expression: `.a |= sub("dogs", "cats")`, expected: []string{ "D0, P[], (!!map)::a: cats are great\n", }, }, { description: "Substitute / Replace string with regex", subdescription: "This uses Golang's regex, described [here](https://github.com/google/re2/wiki/Syntax).\nNote the use of `|=` to run in context of the current string value.", document: "a: cat\nb: heat", expression: `.[] |= sub("(a)", "${1}r")`, expected: []string{ "D0, P[], (!!map)::a: cart\nb: heart\n", }, }, { description: "Custom types: that are really strings", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse cat\nb: !goat heat", expression: `.[] |= sub("(a)", "${1}r")`, expected: []string{ "D0, P[], (!!map)::a: !horse cart\nb: !goat heart\n", }, }, { description: "Split strings", document: `"cat; meow; 1; ; true"`, expression: `split("; ")`, expected: []string{ "D0, P[], (!!seq)::- cat\n- meow\n- \"1\"\n- \"\"\n- \"true\"\n", }, }, { description: "Split strings one match", document: `"word"`, expression: `split("; ")`, expected: []string{ "D0, P[], (!!seq)::- word\n", }, }, { description: "Split splat", skipDoc: true, document: `"word; cat"`, expression: `split("; ")[]`, expected: []string{ "D0, P[0], (!!str)::word\n", "D0, P[1], (!!str)::cat\n", }, }, { skipDoc: true, document: `!horse "word"`, expression: `split("; ")`, expected: []string{ "D0, P[], (!!seq)::- word\n", }, }, { skipDoc: true, document: `""`, expression: `split("; ")`, expected: []string{ "D0, P[], (!!seq)::[]\n", // dont actually want this, just not to error }, }, { skipDoc: true, expression: `split("; ")`, expected: []string{}, }, { description: "To string", subdescription: "Note that you may want to force `yq` to leave scalar values wrapped by passing in `--unwrapScalar=false` or `-r=f`", document: `[1, true, null, ~, cat, {an: object}, [array, 2]]`, expression: ".[] |= to_string", expected: []string{ "D0, P[], (!!seq)::[\"1\", \"true\", \"null\", \"~\", cat, \"{an: object}\", \"[array, 2]\"]\n", }, }, } func TestStringsOperatorScenarios(t *testing.T) { for _, tt := range stringsOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "string-operators", stringsOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_style.go ================================================ package yqlib import ( "container/list" "fmt" ) func parseStyle(customStyle string) (Style, error) { if customStyle == "tagged" { return TaggedStyle, nil } else if customStyle == "double" { return DoubleQuotedStyle, nil } else if customStyle == "single" { return SingleQuotedStyle, nil } else if customStyle == "literal" { return LiteralStyle, nil } else if customStyle == "folded" { return FoldedStyle, nil } else if customStyle == "flow" { return FlowStyle, nil } else if customStyle != "" { return 0, fmt.Errorf("unknown style %v", customStyle) } return 0, nil } func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignStyleOperator: %v") var style Style if !expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { style, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Value) if err != nil { return Context{}, err } } } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("Setting style of : %v", NodeToString(candidate)) if expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { style, err = parseStyle(rhs.MatchingNodes.Front().Value.(*CandidateNode).Value) if err != nil { return Context{}, err } } } candidate.Style = style } return context, nil } func getStyleOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetStyleOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) var style string switch candidate.Style { case TaggedStyle: style = "tagged" case DoubleQuotedStyle: style = "double" case SingleQuotedStyle: style = "single" case LiteralStyle: style = "literal" case FoldedStyle: style = "folded" case FlowStyle: style = "flow" case 0: style = "" default: style = "" } result := candidate.CreateReplacement(ScalarNode, "!!str", style) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_style_test.go ================================================ package yqlib import ( "testing" ) var styleOperatorScenarios = []expressionScenario{ { description: "Update and set style of a particular node (simple)", document: `a: {b: thing, c: something}`, expression: `.a.b = "new" | .a.b style="double"`, expected: []string{ "D0, P[], (!!map)::a: {b: \"new\", c: something}\n", }, }, { description: "Update and set style of a particular node using path variables", document: `a: {b: thing, c: something}`, expression: `with(.a.b ; . = "new" | . style="double")`, expected: []string{ "D0, P[], (!!map)::a: {b: \"new\", c: something}\n", }, }, { description: "Set tagged style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="tagged"`, expected: []string{ "D0, P[], (!!map)::!!map\na: !!str cat\nb: !!int 5\nc: !!float 3.2\ne: !!bool true\nf: !!seq\n - !!int 1\n - !!int 2\n - !!int 3\ng: !!map\n something: !!str cool\n", }, }, { description: "Set double quote style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="double"`, expected: []string{ "D0, P[], (!!map)::a: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\nf:\n - \"1\"\n - \"2\"\n - \"3\"\ng:\n something: \"cool\"\n", }, }, { description: "Set double quote style on map keys too", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `... style="double"`, expected: []string{ "D0, P[], (!!map)::\"a\": \"cat\"\n\"b\": \"5\"\n\"c\": \"3.2\"\n\"e\": \"true\"\n\"f\":\n - \"1\"\n - \"2\"\n - \"3\"\n\"g\":\n \"something\": \"cool\"\n", }, }, { skipDoc: true, document: "bing: &foo {x: z}\na:\n c: cat\n <<: [*foo]", expression: `(... | select(tag=="!!str")) style="single"`, expected: []string{ "D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n", }, }, { description: "Set single quote style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="single"`, expected: []string{ "D0, P[], (!!map)::a: 'cat'\nb: '5'\nc: '3.2'\ne: 'true'\nf:\n - '1'\n - '2'\n - '3'\ng:\n something: 'cool'\n", }, }, { description: "Set literal quote style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="literal"`, expected: []string{ `D0, P[], (!!map)::a: |- cat b: |- 5 c: |- 3.2 e: |- true f: - |- 1 - |- 2 - |- 3 g: something: |- cool `, }, }, { description: "Set folded quote style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="folded"`, expected: []string{ `D0, P[], (!!map)::a: >- cat b: >- 5 c: >- 3.2 e: >- true f: - >- 1 - >- 2 - >- 3 g: something: >- cool `, }, }, { description: "Set flow quote style", document: `{a: cat, b: 5, c: 3.2, e: true, f: [1,2,3], g: { something: cool}}`, expression: `.. style="flow"`, expected: []string{ "D0, P[], (!!map)::{a: cat, b: 5, c: 3.2, e: true, f: [1, 2, 3], g: {something: cool}}\n", }, }, { description: "Reset style - or pretty print", subdescription: "Set empty (default) quote style, note the usage of `...` to match keys too. Note that there is a `--prettyPrint/-P` short flag for this.", dontFormatInputForDoc: true, document: `{a: cat, "b": 5, 'c': 3.2, "e": true, f: [1,2,3], "g": { something: "cool"} }`, expression: `... style=""`, expected: []string{ "D0, P[], (!!map)::a: cat\nb: 5\nc: 3.2\ne: true\nf:\n - 1\n - 2\n - 3\ng:\n something: cool\n", }, }, { description: "Set style relatively with assign-update", document: `{a: single, b: double}`, expression: `.[] style |= .`, expected: []string{ "D0, P[], (!!map)::{a: 'single', b: \"double\"}\n", }, }, { skipDoc: true, document: `{a: cat, b: double}`, expression: `.a style=.b`, expected: []string{ "D0, P[], (!!map)::{a: \"cat\", b: double}\n", }, }, { description: "Read style", document: `{a: "cat", b: 'thing'}`, dontFormatInputForDoc: true, expression: `.. | style`, expected: []string{ "D0, P[], (!!str)::flow\n", "D0, P[a], (!!str)::double\n", "D0, P[b], (!!str)::single\n", }, }, { skipDoc: true, document: `a: cat`, expression: `.. | style`, expected: []string{ "D0, P[], (!!str)::\n", "D0, P[a], (!!str)::\n", }, }, } func TestStyleOperatorScenarios(t *testing.T) { for _, tt := range styleOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "style", styleOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_subtract.go ================================================ package yqlib import ( "fmt" "strconv" "strings" "time" ) func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode { return &ExpressionNode{Operation: &Operation{OperationType: subtractOpType}, LHS: lhs, RHS: rhs} } func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { return compoundAssignFunction(d, context, expressionNode, createSubtractOp) } func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("Subtract operator") return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false) } func subtractArray(lhs *CandidateNode, rhs *CandidateNode) []*CandidateNode { newLHSArray := make([]*CandidateNode, 0) for lindex := 0; lindex < len(lhs.Content); lindex = lindex + 1 { shouldInclude := true for rindex := 0; rindex < len(rhs.Content) && shouldInclude; rindex = rindex + 1 { if recursiveNodeEqual(lhs.Content[lindex], rhs.Content[rindex]) { shouldInclude = false } } if shouldInclude { newLHSArray = append(newLHSArray, lhs.Content[lindex]) } } return newLHSArray } func subtract(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { if lhs.Tag == "!!null" { return lhs.CopyAsReplacement(rhs), nil } target := lhs.CopyWithoutContent() switch lhs.Kind { case MappingNode: return nil, fmt.Errorf("maps not yet supported for subtraction") case SequenceNode: if rhs.Kind != SequenceNode { return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Tag, rhs.GetNicePath(), lhs.Tag) } target.Content = subtractArray(lhs, rhs) case ScalarNode: if rhs.Kind != ScalarNode { return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Tag, rhs.GetNicePath(), lhs.Tag) } target.Kind = ScalarNode target.Style = lhs.Style if err := subtractScalars(context, target, lhs, rhs); err != nil { return nil, err } } return target, nil } func subtractScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { lhsTag := lhs.Tag rhsTag := rhs.Tag lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess lhsTag = lhs.guessTagFromCustomType() lhsIsCustom = true } if !strings.HasPrefix(rhsTag, "!!") { // custom tag - we have to have a guess rhsTag = rhs.guessTagFromCustomType() } isDateTime := lhsTag == "!!timestamp" // if the lhs is a string, it might be a timestamp in a custom format. if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 { _, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value) isDateTime = err == nil } if isDateTime { return subtractDateTime(context.GetDateTimeLayout(), target, lhs, rhs) } else if lhsTag == "!!str" { return fmt.Errorf("strings cannot be subtracted") } else if lhsTag == "!!int" && rhsTag == "!!int" { format, lhsNum, err := parseInt64(lhs.Value) if err != nil { return err } _, rhsNum, err := parseInt64(rhs.Value) if err != nil { return err } result := lhsNum - rhsNum target.Tag = lhs.Tag target.Value = fmt.Sprintf(format, result) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return err } rhsNum, err := strconv.ParseFloat(rhs.Value, 64) if err != nil { return err } result := lhsNum - rhsNum if lhsIsCustom { target.Tag = lhs.Tag } else { target.Tag = "!!float" } target.Value = fmt.Sprintf("%v", result) } else { return fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) } return nil } func subtractDateTime(layout string, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { var durationStr string if strings.HasPrefix(rhs.Value, "-") { durationStr = rhs.Value[1:] } else { durationStr = "-" + rhs.Value } duration, err := time.ParseDuration(durationStr) if err != nil { return fmt.Errorf("unable to parse duration [%v]: %w", rhs.Value, err) } currentTime, err := parseDateTime(layout, lhs.Value) if err != nil { return err } newTime := currentTime.Add(duration) target.Value = newTime.Format(layout) return nil } ================================================ FILE: pkg/yqlib/operator_subtract_test.go ================================================ package yqlib import ( "testing" ) var subtractOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{}`, expression: "(.a - .b) as $x | .", expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, description: "subtract sequence creates a new sequence", expression: `["a", "b"] as $f | {0:$f - ["a"], 1:$f}`, expected: []string{ "D0, P[], (!!map)::0:\n - b\n1:\n - a\n - b\n", }, }, { description: "Array subtraction", expression: `[1,2] - [2,3]`, expected: []string{ "D0, P[], (!!seq)::- 1\n", }, }, { skipDoc: true, expression: `[2,1,2,2] - [2,3]`, expected: []string{ "D0, P[], (!!seq)::- 1\n", }, }, { description: "Array subtraction with nested array", expression: `[[1], 1, 2] - [[1], 3]`, expected: []string{ "D0, P[], (!!seq)::- 1\n- 2\n", }, }, { skipDoc: true, expression: `[[1], 1, [[[2]]]] - [[1], [[[3]]]]`, expected: []string{ "D0, P[], (!!seq)::- 1\n- - - - 2\n", }, }, { description: "Array subtraction with nested object", subdescription: `Note that order of the keys does not matter`, document: `[{a: b, c: d}, {a: b}]`, expression: `. - [{"c": "d", "a": "b"}]`, expected: []string{ "D0, P[], (!!seq)::[{a: b}]\n", }, }, { skipDoc: true, document: `[{a: [1], c: d}, {a: [2], c: d}, {a: b}]`, expression: `. - [{"c": "d", "a": [1]}]`, expected: []string{ "D0, P[], (!!seq)::[{a: [2], c: d}, {a: b}]\n", }, }, { description: "Number subtraction - float", subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", document: `{a: 3, b: 4.5}`, expression: `.a = .a - .b`, expected: []string{ "D0, P[], (!!map)::{a: -1.5, b: 4.5}\n", }, }, { description: "Number subtraction - int", subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", document: `{a: 3, b: 4}`, expression: `.a = .a - .b`, expected: []string{ "D0, P[], (!!map)::{a: -1, b: 4}\n", }, }, { description: "Decrement numbers", document: `{a: 3, b: 5}`, expression: `.[] -= 1`, expected: []string{ "D0, P[], (!!map)::{a: 2, b: 4}\n", }, }, { description: "Date subtraction", subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: 2021-01-01T03:10:00Z`, expression: `.a -= "3h10m"`, expected: []string{ "D0, P[], (!!map)::a: 2021-01-01T00:00:00Z\n", }, }, { description: "Date subtraction - only date", skipDoc: true, document: `a: 2021-01-01`, expression: `.a -= "24h"`, expected: []string{ "D0, P[], (!!map)::a: 2020-12-31T00:00:00Z\n", }, }, { description: "Date subtraction - custom format", subdescription: "Use with_dtf to specify your datetime format. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: Saturday, 15-Dec-01 at 6:00AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, expected: []string{ "D0, P[], (!!map)::a: Saturday, 15-Dec-01 at 2:59AM GMT\n", }, }, { skipDoc: true, description: "Date subtraction - custom format", subdescription: "You can subtract durations from dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: `a: !cat Saturday, 15-Dec-01 at 6:00AM GMT`, expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a -= "3h1m")`, expected: []string{ "D0, P[], (!!map)::a: !cat Saturday, 15-Dec-01 at 2:59AM GMT\n", }, }, { description: "Custom types: that are really numbers", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse 2\nb: !goat 1", expression: `.a -= .b`, expected: []string{ "D0, P[], (!!map)::a: !horse 1\nb: !goat 1\n", }, }, { skipDoc: true, description: "Custom types: that are really floats", subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse 2.5\nb: !goat 1.5", expression: `.a - .b`, expected: []string{ "D0, P[a], (!horse)::1\n", }, }, { skipDoc: true, description: "Custom types: that are really maps", document: `[!horse {a: b, c: d}, !goat {a: b}]`, expression: `. - [{"c": "d", "a": "b"}]`, expected: []string{ "D0, P[], (!!seq)::[!goat {a: b}]\n", }, }, } func TestSubtractOperatorScenarios(t *testing.T) { for _, tt := range subtractOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "subtract", subtractOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_tag.go ================================================ package yqlib import ( "container/list" ) func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("AssignTagOperator: %v") tag := "" if !expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { tag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) log.Debugf("Setting tag of : %v", candidate.GetKey()) if expressionNode.Operation.UpdateAssign { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.RHS) if err != nil { return Context{}, err } if rhs.MatchingNodes.Front() != nil { tag = rhs.MatchingNodes.Front().Value.(*CandidateNode).Value } } candidate.Tag = tag } return context, nil } func getTagOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("GetTagOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) result := candidate.CreateReplacement(ScalarNode, "!!str", candidate.Tag) results.PushBack(result) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_tag_test.go ================================================ package yqlib import ( "testing" ) var tagOperatorScenarios = []expressionScenario{ { description: "tag of key is not a key", subdescription: "so it should have 'a' as the path", skipDoc: true, document: "a: frog\n", expression: `.a | key | tag`, expected: []string{ "D0, P[a], (!!str)::!!str\n", }, }, { description: "Get tag", document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, expression: `.. | tag`, expected: []string{ "D0, P[], (!!str)::!!map\n", "D0, P[a], (!!str)::!!str\n", "D0, P[b], (!!str)::!!int\n", "D0, P[c], (!!str)::!!float\n", "D0, P[e], (!!str)::!!bool\n", "D0, P[f], (!!str)::!!seq\n", }, }, { description: "type is an alias for tag", document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, expression: `.. | type`, expected: []string{ "D0, P[], (!!str)::!!map\n", "D0, P[a], (!!str)::!!str\n", "D0, P[b], (!!str)::!!int\n", "D0, P[c], (!!str)::!!float\n", "D0, P[e], (!!str)::!!bool\n", "D0, P[f], (!!str)::!!seq\n", }, }, { skipDoc: true, document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, expression: `tag`, expected: []string{ "D0, P[], (!!str)::!!map\n", }, }, { skipDoc: true, document: `32`, expression: `. tag= "!!str"`, expected: []string{ "D0, P[], (!!str)::32\n", }, }, { description: "Set custom tag", document: `{a: str}`, expression: `.a tag = "!!mikefarah"`, expected: []string{ "D0, P[], (!!map)::{a: !!mikefarah str}\n", }, }, { skipDoc: true, description: "Set custom type", document: `{a: str}`, expression: `.a type = "!!mikefarah"`, expected: []string{ "D0, P[], (!!map)::{a: !!mikefarah str}\n", }, }, { description: "Find numbers and convert them to strings", document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `(.. | select(tag == "!!int")) tag= "!!str"`, expected: []string{ "D0, P[], (!!map)::{a: cat, b: \"5\", c: 3.2, e: true}\n", }, }, { skipDoc: true, document: `{a: "!!frog", b: "!!customTag"}`, expression: `.[] tag |= .`, expected: []string{ "D0, P[], (!!map)::{a: !!frog \"!!frog\", b: !!customTag \"!!customTag\"}\n", }, }, } func TestTagOperatorScenarios(t *testing.T) { for _, tt := range tagOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "tag", tagOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_to_number.go ================================================ package yqlib import ( "container/list" "fmt" "strconv" ) func tryConvertToNumber(value string) (string, bool) { // try an int first _, _, err := parseInt64(value) if err == nil { return "!!int", true } // try float _, floatErr := strconv.ParseFloat(value, 64) if floatErr == nil { return "!!float", true } return "", false } func toNumberOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { log.Debugf("ToNumberOperator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != ScalarNode { return Context{}, fmt.Errorf("cannot convert node at path %v of tag %v to number", candidate.GetNicePath(), candidate.Tag) } if candidate.Tag == "!!int" || candidate.Tag == "!!float" { // it already is a number! results.PushBack(candidate) } else { tag, converted := tryConvertToNumber(candidate.Value) if converted { result := candidate.CreateReplacement(ScalarNode, tag, candidate.Value) results.PushBack(result) } else { return Context{}, fmt.Errorf("cannot convert node value [%v] at path %v of tag %v to number", candidate.Value, candidate.GetNicePath(), candidate.Tag) } } } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_to_number_test.go ================================================ package yqlib import ( "testing" ) var toNumberScenarios = []expressionScenario{ { description: "Converts strings to numbers", document: `["3", "3.1", "-1e3"]`, expression: `.[] | to_number`, expected: []string{ "D0, P[0], (!!int)::3\n", "D0, P[1], (!!float)::3.1\n", "D0, P[2], (!!float)::-1e3\n", }, }, { skipDoc: true, description: "Converts strings to numbers, with tonumber because jq", document: `["3", "3.1", "-1e3"]`, expression: `.[] | tonumber`, expected: []string{ "D0, P[0], (!!int)::3\n", "D0, P[1], (!!float)::3.1\n", "D0, P[2], (!!float)::-1e3\n", }, }, { description: "Doesn't change numbers", document: `[3, 3.1, -1e3]`, expression: `.[] | to_number`, expected: []string{ "D0, P[0], (!!int)::3\n", "D0, P[1], (!!float)::3.1\n", "D0, P[2], (!!float)::-1e3\n", }, }, { description: "Cannot convert null", expression: `.a.b | to_number`, expectedError: "cannot convert node value [null] at path a.b of tag !!null to number", }, } func TestToNumberOperatorScenarios(t *testing.T) { for _, tt := range toNumberScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "to_number", toNumberScenarios) } ================================================ FILE: pkg/yqlib/operator_traverse_path.go ================================================ package yqlib import ( "container/list" "fmt" "slices" "github.com/elliotchance/orderedmap" ) type traversePreferences struct { DontFollowAlias bool IncludeMapKeys bool DontAutoCreate bool // by default, we automatically create entries on the fly. DontIncludeMapValues bool OptionalTraverse bool // e.g. .adf? ExactKeyMatch bool // by default we let wild/glob patterns. Don't do that for merge though. } func splat(context Context, prefs traversePreferences) (Context, error) { return traverseNodesWithArrayIndices(context, make([]*CandidateNode, 0), prefs) } func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("traversePathOperator") var matches = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { newNodes, err := traverse(context, el.Value.(*CandidateNode), expressionNode.Operation) if err != nil { return Context{}, err } matches.PushBackList(newNodes) } return context.ChildContext(matches), nil } func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) { log.Debug("Traversing %v", NodeToString(matchingNode)) if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate { log.Debugf("Guessing kind") // we must have added this automatically, lets guess what it should be now switch operation.Value.(type) { case int, int64: log.Debugf("probably an array") matchingNode.Kind = SequenceNode default: log.Debugf("probably a map") matchingNode.Kind = MappingNode } matchingNode.Tag = "" } switch matchingNode.Kind { case MappingNode: log.Debug("its a map with %v entries", len(matchingNode.Content)/2) return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false) case SequenceNode: log.Debug("its a sequence of %v things!", len(matchingNode.Content)) return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences)) case AliasNode: log.Debug("its an alias!") matchingNode = matchingNode.Alias return traverse(context, matchingNode, operation) default: return list.New(), nil } } func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { //lhs may update the variable context, we should pass that into the RHS // BUT we still return the original context back (see jq) // https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|... log.Debugf("--traverseArrayOperator") if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType { return sliceArrayOperator(d, context, expressionNode.RHS.RHS) } lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } // rhs is a collect expression that will yield indices to retrieve of the arrays rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) if err != nil { return Context{}, err } prefs := traversePreferences{} if expressionNode.Operation.Preferences != nil { prefs = expressionNode.Operation.Preferences.(traversePreferences) } var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Content log.Debugf("indicesToTraverse %v", len(indicesToTraverse)) //now we traverse the result of the lhs against the indices we found result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs) if err != nil { return Context{}, err } return context.ChildContext(result.MatchingNodes), nil } func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*CandidateNode, prefs traversePreferences) (Context, error) { var matchingNodeMap = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) newNodes, err := traverseArrayIndices(context, candidate, indicesToTraverse, prefs) if err != nil { return Context{}, err } matchingNodeMap.PushBackList(newNodes) } return context.ChildContext(matchingNodeMap), nil } func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse if matchingNode.Tag == "!!null" { log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array") // auto vivification matchingNode.Tag = "" matchingNode.Kind = SequenceNode //check that the indices are numeric, if not, then we should create an object if len(indicesToTraverse) != 0 && indicesToTraverse[0].Tag != "!!int" { matchingNode.Kind = MappingNode } } switch matchingNode.Kind { case AliasNode: matchingNode = matchingNode.Alias return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs) case SequenceNode: return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs) case MappingNode: return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs) } log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, matchingNode.Tag) return list.New(), nil } func traverseMapWithIndices(context Context, candidate *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) { if len(indices) == 0 { return traverseMap(context, candidate, createStringScalarNode(""), prefs, true) } var matchingNodeMap = list.New() for _, indexNode := range indices { log.Debug("traverseMapWithIndices: %v", indexNode.Value) newNodes, err := traverseMap(context, candidate, indexNode, prefs, false) if err != nil { return nil, err } matchingNodeMap.PushBackList(newNodes) } return matchingNodeMap, nil } func traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, prefs traversePreferences) (*list.List, error) { log.Debug("traverseArrayWithIndices") var newMatches = list.New() if len(indices) == 0 { log.Debug("splatting") var index int for index = 0; index < len(node.Content); index = index + 1 { newMatches.PushBack(node.Content[index]) } return newMatches, nil } for _, indexNode := range indices { log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value) index, err := parseInt(indexNode.Value) if err != nil && prefs.OptionalTraverse { continue } if err != nil { return nil, fmt.Errorf("cannot index array with '%v' (%w)", indexNode.Value, err) } indexToUse := index contentLength := len(node.Content) for contentLength <= index { if contentLength == 0 { // default to nice yaml formatting node.Style = 0 } valueNode := createScalarNode(nil, "null") node.AddChild(valueNode) contentLength = len(node.Content) } if indexToUse < 0 { indexToUse = contentLength + indexToUse } if indexToUse < 0 { return nil, fmt.Errorf("index [%v] out of range, array size is %v", index, contentLength) } newMatches.PushBack(node.Content[indexToUse]) } return newMatches, nil } func keyMatches(key *CandidateNode, wantedKey string, exactKeyMatch bool) bool { if exactKeyMatch { // this is used for merge return key.Value == wantedKey } return matchKey(key.Value, wantedKey) } func traverseMap(context Context, matchingNode *CandidateNode, keyNode *CandidateNode, prefs traversePreferences, splat bool) (*list.List, error) { var newMatches = orderedmap.NewOrderedMap() err := doTraverseMap(newMatches, matchingNode, keyNode.Value, prefs, splat) if err != nil { return nil, err } if !splat && !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 { log.Debugf("no matches, creating one for %v", NodeToString(keyNode)) //no matches, create one automagically valueNode := matchingNode.CreateChild() valueNode.Kind = ScalarNode valueNode.Tag = "!!null" valueNode.Value = "null" if len(matchingNode.Content) == 0 { matchingNode.Style = 0 } keyNode, valueNode = matchingNode.AddKeyValueChild(keyNode, valueNode) if prefs.IncludeMapKeys { newMatches.Set(keyNode.GetKey(), keyNode) } if !prefs.DontIncludeMapValues { newMatches.Set(valueNode.GetKey(), valueNode) } } results := list.New() i := 0 for el := newMatches.Front(); el != nil; el = el.Next() { results.PushBack(el.Value) i++ } return results, nil } func doTraverseMap(newMatches *orderedmap.OrderedMap, node *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { // value.Content is a concatenated array of key, value, // so keys are in the even indices, values in odd. // merge aliases are defined first, but we only want to traverse them // if we don't find a match directly on this node first. var contents = node.Content if !prefs.DontFollowAlias { if ConfiguredYamlPreferences.FixMergeAnchorToSpec { // First evaluate merge keys to make explicit keys take precedence, following spec // We also iterate in reverse to make earlier merge keys take precedence, // although normally there's just one '<<' for index := len(node.Content) - 2; index >= 0; index -= 2 { keyNode := node.Content[index] valueNode := node.Content[index+1] if keyNode.Tag == "!!merge" { log.Debug("Merge anchor") err := traverseMergeAnchor(newMatches, valueNode, wantedKey, prefs, splat) if err != nil { return err } } } } } for index := 0; index+1 < len(contents); index = index + 2 { key := contents[index] value := contents[index+1] //skip the 'merge' tag, find a direct match first if key.Tag == "!!merge" && !prefs.DontFollowAlias && wantedKey != key.Value { if !ConfiguredYamlPreferences.FixMergeAnchorToSpec { log.Debug("Merge anchor") if showMergeAnchorToSpecWarning { log.Warning("--yaml-fix-merge-anchor-to-spec is false; causing merge anchors to override the existing values which isn't to the yaml spec. This flag will default to true in late 2025. See https://mikefarah.gitbook.io/yq/operators/traverse-read for more details.") showMergeAnchorToSpecWarning = false } err := traverseMergeAnchor(newMatches, value, wantedKey, prefs, splat) if err != nil { return err } } } else if splat || keyMatches(key, wantedKey, prefs.ExactKeyMatch) { log.Debug("MATCHED") if prefs.IncludeMapKeys { log.Debug("including key") keyName := key.GetKey() if !newMatches.Set(keyName, key) { log.Debug("overwriting existing key") } } if !prefs.DontIncludeMapValues { log.Debug("including value") valueName := value.GetKey() if !newMatches.Set(valueName, value) { log.Debug("overwriting existing value") } } } } return nil } func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { if merge.Kind == AliasNode { merge = merge.Alias } switch merge.Kind { case MappingNode: return doTraverseMap(newMatches, merge, wantedKey, prefs, splat) case SequenceNode: content := slices.All(merge.Content) if ConfiguredYamlPreferences.FixMergeAnchorToSpec { // Reverse to make earlier values take precedence, following spec content = slices.Backward(merge.Content) } for _, childValue := range content { if childValue.Kind == AliasNode { childValue = childValue.Alias } if childValue.Kind != MappingNode { log.Debugf( "can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", childValue.Tag) return nil } err := doTraverseMap(newMatches, childValue, wantedKey, prefs, splat) if err != nil { return err } } return nil default: log.Debugf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got %v", merge.Tag) return nil } } func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) { log.Debug("operation Value %v", operation.Value) indices := []*CandidateNode{{Value: operation.StringValue}} return traverseArrayWithIndices(candidate, indices, prefs) } ================================================ FILE: pkg/yqlib/operator_traverse_path_test.go ================================================ package yqlib import ( "testing" ) var mergeDocSample = `foo: &foo a: foo_a thing: foo_thing c: foo_c bar: &bar b: bar_b thing: bar_thing c: bar_c foobarList: b: foobarList_b <<: [*foo,*bar] c: foobarList_c foobar: c: foobar_c <<: *foo thing: foobar_thing ` var fixedTraversePathOperatorScenarios = []expressionScenario{ { description: "FIXED: Traversing merge anchors with override", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour.", document: mergeDocSample, expression: `.foobar.c`, expected: []string{ "D0, P[foobar c], (!!str)::foobar_c\n", }, }, { description: "FIXED: Traversing merge anchor lists", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ "D0, P[foo thing], (!!str)::foo_thing\n", }, }, { description: "FIXED: Splatting merge anchors", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobar[]`, expected: []string{ "D0, P[foo a], (!!str)::foo_a\n", "D0, P[foobar thing], (!!str)::foobar_thing\n", "D0, P[foobar c], (!!str)::foobar_c\n", }, }, { description: "FIXED: Splatting merge anchor lists", subdescription: "Set `--yaml-fix-merge-anchor-to-spec=true` to get this correct merge behaviour. Note that the keys earlier in the merge anchors sequence override later ones", document: mergeDocSample, expression: `.foobarList[]`, expected: []string{ "D0, P[foobarList b], (!!str)::foobarList_b\n", "D0, P[foo thing], (!!str)::foo_thing\n", "D0, P[foobarList c], (!!str)::foobarList_c\n", "D0, P[foo a], (!!str)::foo_a\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList.b`, expected: []string{ "D0, P[foobarList b], (!!str)::foobarList_b\n", }, }, } var badTraversePathOperatorScenarios = []expressionScenario{ { description: "LEGACY: Traversing merge anchors with override", subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar.c`, expected: []string{ "D0, P[foo c], (!!str)::foo_c\n", }, }, { description: "LEGACY: Traversing merge anchor lists", subdescription: "Note that the later merge anchors override previous, " + "but this is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList.thing`, expected: []string{ "D0, P[bar thing], (!!str)::bar_thing\n", }, }, { description: "LEGACY: Splatting merge anchors", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobar[]`, expected: []string{ "D0, P[foo c], (!!str)::foo_c\n", "D0, P[foo a], (!!str)::foo_a\n", "D0, P[foobar thing], (!!str)::foobar_thing\n", }, }, { description: "LEGACY: Splatting merge anchor lists", subdescription: "With legacy override behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList[]`, expected: []string{ "D0, P[bar b], (!!str)::bar_b\n", "D0, P[foo a], (!!str)::foo_a\n", "D0, P[bar thing], (!!str)::bar_thing\n", "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, { skipDoc: true, subdescription: "This is legacy behaviour, see --yaml-fix-merge-anchor-to-spec", document: mergeDocSample, expression: `.foobarList.b`, expected: []string{ "D0, P[bar b], (!!str)::bar_b\n", }, }, } var traversePathOperatorScenarios = []expressionScenario{ { skipDoc: true, description: "strange map with key but no value", document: "!!null\n-", expression: ".x", expected: []string{ "D0, P[x], (!!null)::null\n", }, skipForGoccy: true, // throws an error instead, that's fine }, { skipDoc: true, description: "access merge anchors", document: "foo: &foo {x: y}\nbar:\n <<: *foo\n", expression: `.bar["<<"] | alias`, expected: []string{ "D0, P[bar <<], (!!str)::foo\n", }, }, { skipDoc: true, description: "dynamically set parent and key", expression: `.a.b.c = 3 | .a.b.c`, expected: []string{ "D0, P[a b c], (!!int)::3\n", }, }, { skipDoc: true, description: "dynamically set parent and key in array", expression: `.a.b[0] = 3 | .a.b[0]`, expected: []string{ "D0, P[a b 0], (!!int)::3\n", }, }, { skipDoc: true, description: "dynamically set parent and key", expression: `.a.b = ["x","y"] | .a.b[1]`, expected: []string{ "D0, P[a b 1], (!!str)::y\n", }, }, { skipDoc: true, description: "splat empty map", document: "{}", expression: ".[]", expected: []string{}, }, { skipDoc: true, document: `[[1]]`, expression: `.[0][0]`, expected: []string{ "D0, P[0 0], (!!int)::1\n", }, }, { skipDoc: true, expression: `.cat["12"] = "things"`, expected: []string{ "D0, P[], ()::cat:\n \"12\": things\n", }, }, { skipDoc: true, document: `blah: {}`, expression: `.blah.cat = "cool"`, expected: []string{ "D0, P[], (!!map)::blah:\n cat: cool\n", }, }, { skipDoc: true, document: `blah: []`, expression: `.blah.0 = "cool"`, expected: []string{ "D0, P[], (!!map)::blah:\n - cool\n", }, }, { skipDoc: true, document: `b: cat`, expression: ".b\n", expected: []string{ "D0, P[b], (!!str)::cat\n", }, }, { skipDoc: true, document: `[[[1]]]`, expression: `.[0][0][0]`, expected: []string{ "D0, P[0 0 0], (!!int)::1\n", }, }, { skipDoc: true, expression: `.["cat"] = "thing"`, expected: []string{ "D0, P[], ()::cat: thing\n", }, }, { description: "Simple map navigation", document: `{a: {b: apple}}`, expression: `.a`, expected: []string{ "D0, P[a], (!!map)::{b: apple}\n", }, }, { description: "Splat", subdescription: "Often used to pipe children into other operators", document: `[{b: apple}, {c: banana}]`, expression: `.[]`, expected: []string{ "D0, P[0], (!!map)::{b: apple}\n", "D0, P[1], (!!map)::{c: banana}\n", }, }, { description: "Optional Splat", subdescription: "Just like splat, but won't error if you run it against scalars", document: `"cat"`, expression: `.[]`, expected: []string{}, }, { description: "Special characters", subdescription: "Use quotes with square brackets around path elements with special characters", document: `{"{}": frog}`, expression: `.["{}"]`, expected: []string{ "D0, P[{}], (!!str)::frog\n", }, }, { description: "Nested special characters", document: `a: {"key.withdots": {"another.key": apple}}`, expression: `.a["key.withdots"]["another.key"]`, expected: []string{ "D0, P[a key.withdots another.key], (!!str)::apple\n", }, }, { description: "Keys with spaces", subdescription: "Use quotes with square brackets around path elements with special characters", document: `{"red rabbit": frog}`, expression: `.["red rabbit"]`, expected: []string{ "D0, P[red rabbit], (!!str)::frog\n", }, }, { skipDoc: true, document: `{"flying fox": frog}`, expression: `.["flying fox"]`, expected: []string{ "D0, P[flying fox], (!!str)::frog\n", }, }, { skipDoc: true, document: `c: dog`, expression: `.[.a.b] as $x | .`, expected: []string{ "D0, P[], (!!map)::c: dog\n", }, }, { description: "Dynamic keys", subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`, document: `{b: apple, apple: crispy yum, banana: soft yum}`, expression: `.[.b]`, expected: []string{ "D0, P[apple], (!!str)::crispy yum\n", }, }, { skipDoc: true, document: `{b: apple, fruit: {apple: yum, banana: smooth}}`, expression: `.fruit[.b]`, expected: []string{ "D0, P[fruit apple], (!!str)::yum\n", }, }, { description: "Children don't exist", subdescription: "Nodes are added dynamically while traversing", document: `{c: banana}`, expression: `.a.b`, expected: []string{ "D0, P[a b], (!!null)::null\n", }, }, { description: "Optional identifier", subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected", document: `[1,2,3]`, expression: `.a?`, expected: []string{}, }, { skipDoc: true, document: `[[1,2,3], {a: frog}]`, expression: `.[] | .["a"]?`, expected: []string{"D0, P[1 a], (!!str)::frog\n"}, }, { skipDoc: true, document: ``, expression: `.[1].a`, expected: []string{ "D0, P[1 a], (!!null)::null\n", }, }, { skipDoc: true, document: `{}`, expression: `.a[1]`, expected: []string{ "D0, P[a 1], (!!null)::null\n", }, }, { description: "Wildcard matching", document: `{a: {cat: apple, mad: things}}`, expression: `.a."*a*"`, expected: []string{ "D0, P[a cat], (!!str)::apple\n", "D0, P[a mad], (!!str)::things\n", }, }, { skipDoc: true, document: `{a: {cat: {b: 3}, mad: {b: 4}, fad: {c: t}}}`, expression: `.a."*a*".b`, expected: []string{ "D0, P[a cat b], (!!int)::3\n", "D0, P[a mad b], (!!int)::4\n", "D0, P[a fad b], (!!null)::null\n", }, }, { skipDoc: true, document: `{a: {cat: apple, mad: things}}`, expression: `.a | (.cat, .mad)`, expected: []string{ "D0, P[a cat], (!!str)::apple\n", "D0, P[a mad], (!!str)::things\n", }, }, { skipDoc: true, document: `{a: {cat: apple, mad: things}}`, expression: `.a | (.cat, .mad, .fad)`, expected: []string{ "D0, P[a cat], (!!str)::apple\n", "D0, P[a mad], (!!str)::things\n", "D0, P[a fad], (!!null)::null\n", }, }, { skipDoc: true, document: `{a: {cat: apple, mad: things}}`, expression: `.a | (.cat, .mad, .fad) | select( (. == null) | not)`, expected: []string{ "D0, P[a cat], (!!str)::apple\n", "D0, P[a mad], (!!str)::things\n", }, }, { description: "Aliases", document: `{a: &cat {c: frog}, b: *cat}`, expression: `.b`, expected: []string{ "D0, P[b], (alias)::*cat\n", }, }, { description: "Traversing aliases with splat", document: `{a: &cat {c: frog}, b: *cat}`, expression: `.b[]`, expected: []string{ "D0, P[a c], (!!str)::frog\n", }, }, { description: "Traversing aliases explicitly", document: `{a: &cat {c: frog}, b: *cat}`, expression: `.b.c`, expected: []string{ "D0, P[a c], (!!str)::frog\n", }, }, { description: "Traversing arrays by index", document: `[1,2,3]`, expression: `.[0]`, expected: []string{ "D0, P[0], (!!int)::1\n", }, }, { description: "Traversing nested arrays by index", dontFormatInputForDoc: true, document: `[[], [cat]]`, expression: `.[1][0]`, expected: []string{ "D0, P[1 0], (!!str)::cat\n", }, }, { description: "Maps with numeric keys", document: `{2: cat}`, expression: `.[2]`, expected: []string{ "D0, P[2], (!!str)::cat\n", }, }, { description: "Maps with non existing numeric keys", document: `{a: b}`, expression: `.[0]`, expected: []string{ "D0, P[0], (!!null)::null\n", }, }, { skipDoc: true, description: "Merge anchor with inline map", document: `{<<: {a: 42}}`, expression: `.a`, expected: []string{ "D0, P[<< a], (!!int)::42\n", }, }, { skipDoc: true, description: "Merge anchor with sequence with inline map", document: `{<<: [{a: 42}]}`, expression: `.a`, expected: []string{ "D0, P[<< 0 a], (!!int)::42\n", }, }, { skipDoc: true, description: "Merge anchor with aliased sequence with inline map", document: `{s: &s [{a: 42}], m: {<<: *s}}`, expression: `.m.a`, expected: []string{ "D0, P[s 0 a], (!!int)::42\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobar`, expected: []string{ "D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n", }, }, { description: "Traversing merge anchors", document: mergeDocSample, expression: `.foobar.a`, expected: []string{ "D0, P[foo a], (!!str)::foo_a\n", }, }, { description: "Traversing merge anchors with local override", document: mergeDocSample, expression: `.foobar.thing`, expected: []string{ "D0, P[foobar thing], (!!str)::foobar_thing\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList`, expected: []string{ "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList.a`, expected: []string{ "D0, P[foo a], (!!str)::foo_a\n", }, }, { skipDoc: true, document: mergeDocSample, expression: `.foobarList.c`, expected: []string{ "D0, P[foobarList c], (!!str)::foobarList_c\n", }, }, { skipDoc: true, document: `[a,b,c]`, expression: `.[]`, expected: []string{ "D0, P[0], (!!str)::a\n", "D0, P[1], (!!str)::b\n", "D0, P[2], (!!str)::c\n", }, }, { skipDoc: true, document: `[a,b,c]`, expression: `[]`, expected: []string{ "D0, P[], (!!seq)::[]\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[0]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", }, }, { description: "Select multiple indices", document: `{a: [a,b,c]}`, expression: `.a[0, 2]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[0, 2]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[-1]`, expected: []string{ "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[-2]`, expected: []string{ "D0, P[a 1], (!!str)::b\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", "D0, P[a 1], (!!str)::b\n", "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a[]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", "D0, P[a 1], (!!str)::b\n", "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, document: `{a: [a,b,c]}`, expression: `.a | .[]`, expected: []string{ "D0, P[a 0], (!!str)::a\n", "D0, P[a 1], (!!str)::b\n", "D0, P[a 2], (!!str)::c\n", }, }, { skipDoc: true, description: "Duplicate keys", subdescription: "outside merge anchor", document: `{a: 1, a: 2}`, expression: `.a`, expected: []string{ "D0, P[a], (!!int)::2\n", }, }, { skipDoc: true, description: "Traversing map with invalid merge anchor should not fail", subdescription: "Otherwise code cannot do anything with it", document: `{a: 42, <<: 37}`, expression: `.a`, expected: []string{ "D0, P[a], (!!int)::42\n", }, }, { skipDoc: true, description: "Directly accessing invalid merge anchor should not fail", document: `{<<: 37}`, expression: `.<<`, expected: []string{ "D0, P[<<], (!!int)::37\n", }, }, { skipDoc: true, description: "!!str << should not be treated as merge anchor", document: `{!!str <<: {a: 37}}`, expression: `.a`, expected: []string{ "D0, P[a], (!!null)::null\n", }, }, } func TestTraversePathOperatorScenarios(t *testing.T) { for _, tt := range append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...) { testScenario(t, &tt) } documentOperatorScenarios(t, "traverse-read", append(traversePathOperatorScenarios, badTraversePathOperatorScenarios...)) } func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) { ConfiguredYamlPreferences.FixMergeAnchorToSpec = true for _, tt := range append(fixedTraversePathOperatorScenarios, traversePathOperatorScenarios...) { testScenario(t, &tt) } appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios) ConfiguredYamlPreferences.FixMergeAnchorToSpec = false } ================================================ FILE: pkg/yqlib/operator_union.go ================================================ package yqlib import "container/list" func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("unionOperator--") log.Debug("unionOperator: context: %v", NodesToString(context.MatchingNodes)) lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } log.Debug("unionOperator: lhs: %v", NodesToString(lhs.MatchingNodes)) log.Debug("unionOperator: rhs input: %v", NodesToString(context.MatchingNodes)) log.Debug("unionOperator: rhs: %v", expressionNode.RHS.Operation.toString()) rhs, err := d.GetMatchingNodes(context, expressionNode.RHS) if err != nil { return Context{}, err } log.Debug("unionOperator: lhs: %v", lhs.ToString()) log.Debug("unionOperator: rhs: %v", rhs.ToString()) results := lhs.ChildContext(list.New()) for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) results.MatchingNodes.PushBack(node) } // this can happen when both expressions modify the context // instead of creating their own. /// (.foo = "bar"), (.thing = "cat") if rhs.MatchingNodes != lhs.MatchingNodes { for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) log.Debug("union operator rhs: processing %v", NodeToString(node)) results.MatchingNodes.PushBack(node) } } log.Debug("union operator: all together: %v", results.ToString()) return results, nil } ================================================ FILE: pkg/yqlib/operator_union_test.go ================================================ package yqlib import ( "testing" ) var unionOperatorScenarios = []expressionScenario{ { skipDoc: true, document: "{}", expression: `(.a, .b.c) as $x | .`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, description: "clone test", expression: `"abc" as $a | [$a, "cat"]`, expected: []string{ "D0, P[], (!!seq)::- abc\n- cat\n", }, }, { skipDoc: true, expression: `(.foo = "bar"), (.toe = "jam")`, expected: []string{ "D0, P[], ()::foo: bar\ntoe: jam\n", }, }, { description: "Combine scalars", expression: `1, true, "cat"`, expected: []string{ "D0, P[], (!!int)::1\n", "D0, P[], (!!bool)::true\n", "D0, P[], (!!str)::cat\n", }, }, { description: "Combine selected paths", document: `{a: fieldA, b: fieldB, c: fieldC}`, expression: `.a, .c`, expected: []string{ "D0, P[a], (!!str)::fieldA\n", "D0, P[c], (!!str)::fieldC\n", }, }, } func TestUnionOperatorScenarios(t *testing.T) { for _, tt := range unionOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "union", unionOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_unique.go ================================================ package yqlib import ( "container/list" "fmt" "github.com/elliotchance/orderedmap" ) func unique(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} uniqueByExpression := &ExpressionNode{Operation: &Operation{OperationType: uniqueByOpType}, RHS: selfExpression} return uniqueBy(d, context, uniqueByExpression) } func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("uniqueBy Operator") var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) if candidate.Kind != SequenceNode { return Context{}, fmt.Errorf("only arrays are supported for unique") } var newMatches = orderedmap.NewOrderedMap() for _, child := range candidate.Content { rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.RHS) if err != nil { return Context{}, err } keyValue, err := getUniqueKeyValue(rhs) if err != nil { return Context{}, err } _, exists := newMatches.Get(keyValue) if !exists { newMatches.Set(keyValue, child) } } resultNode := candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style) for el := newMatches.Front(); el != nil; el = el.Next() { resultNode.AddChild(el.Value.(*CandidateNode)) } results.PushBack(resultNode) } return context.ChildContext(results), nil } func getUniqueKeyValue(rhs Context) (string, error) { keyValue := "null" var err error if rhs.MatchingNodes.Len() > 0 { first := rhs.MatchingNodes.Front() keyCandidate := first.Value.(*CandidateNode) keyValue = keyCandidate.Value if keyCandidate.Kind != ScalarNode { keyValue, err = encodeToString(keyCandidate, encoderPreferences{YamlFormat, 0}) } } return keyValue, err } ================================================ FILE: pkg/yqlib/operator_unique_test.go ================================================ package yqlib import ( "testing" ) var uniqueOperatorScenarios = []expressionScenario{ { description: "Unique array of scalars (string/numbers)", subdescription: "Note that unique maintains the original order of the array.", document: `[2,1,3,2]`, expression: `unique`, expected: []string{ "D0, P[], (!!seq)::[2, 1, 3]\n", }, }, { description: "Unique splat", skipDoc: true, document: `[2,1,2]`, expression: `unique[]`, expected: []string{ "D0, P[0], (!!int)::2\n", "D0, P[1], (!!int)::1\n", }, }, { description: "Unique nulls", subdescription: "Unique works on the node value, so it considers different representations of nulls to be different", document: `[~,null, ~, null]`, expression: `unique`, expected: []string{ "D0, P[], (!!seq)::[~, null]\n", }, }, { description: "Unique all nulls", subdescription: "Run against the node tag to unique all the nulls", document: `[~,null, ~, null]`, expression: `unique_by(tag)`, expected: []string{ "D0, P[], (!!seq)::[~]\n", }, }, { description: "Unique array objects", document: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: cat}]`, expression: `unique`, expected: []string{ "D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n", }, }, { description: "Unique array of objects by a field", document: `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`, expression: `unique_by(.name)`, expected: []string{ "D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n", }, }, { description: "Unique array of arrays", document: `[[cat,dog], [cat, sheep], [cat,dog]]`, expression: `unique`, expected: []string{ "D0, P[], (!!seq)::[[cat, dog], [cat, sheep]]\n", }, }, { skipDoc: true, document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, expression: `unique_by(.name)`, expected: []string{ "D0, P[], (!!seq)::[{name: harry, pet: cat}, {pet: fish}]\n", }, }, { description: "unique by splat", skipDoc: true, document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, expression: `unique_by(.name)[]`, expected: []string{ "D0, P[0], (!!map)::{name: harry, pet: cat}\n", "D0, P[1], (!!map)::{pet: fish}\n", }, }, { skipDoc: true, document: `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`, expression: `unique_by(.cat.dog)`, expected: []string{ "D0, P[], (!!seq)::[{name: harry, pet: cat}]\n", }, }, { skipDoc: true, document: "# abc\n[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]\n# xyz", expression: `unique_by(.name)`, expected: []string{ "D0, P[], (!!seq)::# abc\n[{name: harry, pet: cat}, {pet: fish}]\n# xyz\n", }, skipForGoccy: true, // https://github.com/goccy/go-yaml/issues/757 }, } func TestUniqueOperatorScenarios(t *testing.T) { for _, tt := range uniqueOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "unique", uniqueOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_value.go ================================================ package yqlib import "container/list" func referenceOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil } func valueOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("value = %v", expressionNode.Operation.CandidateNode.Value) if context.MatchingNodes.Len() == 0 { clone := expressionNode.Operation.CandidateNode.Copy() return context.SingleChildContext(clone), nil } var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { clone := expressionNode.Operation.CandidateNode.Copy() results.PushBack(clone) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_value_test.go ================================================ package yqlib import ( "testing" ) var valueOperatorScenarios = []expressionScenario{ { document: ``, expression: `1`, expected: []string{ "D0, P[], (!!int)::1\n", }, }, { document: `[1,2,3]`, expression: `.[] | "foo"`, expected: []string{ "D0, P[], (!!str)::foo\n", "D0, P[], (!!str)::foo\n", "D0, P[], (!!str)::foo\n", }, }, { document: `[1,2,3]`, expression: `[.[] | "foo"] | .[0] = "cat"`, expected: []string{ "D0, P[], (!!seq)::- cat\n- foo\n- foo\n", }, }, { expression: `"foo"`, expected: []string{ "D0, P[], (!!str)::foo\n", }, }, { document: ``, expression: `0x9f`, expected: []string{ "D0, P[], (!!int)::0x9f\n", }, }, { document: ``, expression: `0x1A`, expected: []string{ "D0, P[], (!!int)::0x1A\n", }, }, { document: ``, expression: `0x1A + 2`, expected: []string{ "D0, P[], (!!int)::0x1C\n", }, }, { document: ``, expression: `0x12 * 2`, expected: []string{ "D0, P[], (!!int)::0x24\n", }, }, { document: ``, expression: `0xF - 1`, expected: []string{ "D0, P[], (!!int)::0xE\n", }, }, { document: ``, expression: `12`, expected: []string{ "D0, P[], (!!int)::12\n", }, }, { document: ``, expression: `12 + 2`, expected: []string{ "D0, P[], (!!int)::14\n", }, }, { document: ``, expression: `12 * 2`, expected: []string{ "D0, P[], (!!int)::24\n", }, }, { document: ``, expression: `12 - 2`, expected: []string{ "D0, P[], (!!int)::10\n", }, }, { document: ``, expression: `0X12`, expected: []string{ "D0, P[], (!!int)::0X12\n", }, }, { document: ``, expression: `-1`, expected: []string{ "D0, P[], (!!int)::-1\n", }, }, { document: ``, expression: `1.2`, expected: []string{ "D0, P[], (!!float)::1.2\n", }, }, { document: ``, expression: `-5.2e11`, expected: []string{ "D0, P[], (!!float)::-5.2e11\n", }, }, { document: ``, expression: `5e-10`, expected: []string{ "D0, P[], (!!float)::5e-10\n", }, }, { document: ``, expression: `"cat"`, expected: []string{ "D0, P[], (!!str)::cat\n", }, }, { document: ``, expression: `"frog jumps"`, expected: []string{ "D0, P[], (!!str)::frog jumps\n", }, }, { document: ``, expression: `"1.3"`, expected: []string{ "D0, P[], (!!str)::1.3\n", }, }, { document: ``, expression: `"true"`, expected: []string{ "D0, P[], (!!str)::true\n", }, }, { document: ``, expression: `true`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { document: ``, expression: `false`, expected: []string{ "D0, P[], (!!bool)::false\n", }, }, { document: ``, expression: `Null`, expected: []string{ "D0, P[], (!!null)::Null\n", }, }, { document: ``, expression: `~`, expected: []string{ "D0, P[], (!!null)::~\n", }, }, } func TestValueOperatorScenarios(t *testing.T) { for _, tt := range valueOperatorScenarios { testScenario(t, &tt) } } ================================================ FILE: pkg/yqlib/operator_variables.go ================================================ package yqlib import ( "container/list" "fmt" ) func getVariableOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { variableName := expressionNode.Operation.StringValue log.Debug("getVariableOperator %v", variableName) result := context.GetVariable(variableName) if result == nil { result = list.New() } return context.ChildContext(result), nil } type assignVarPreferences struct { IsReference bool } func useWithPipe(_ *dataTreeNavigator, _ Context, _ *ExpressionNode) (Context, error) { return Context{}, fmt.Errorf("must use variable with a pipe, e.g. `exp as $x | ...`") } // variables are like loops in jq // https://stedolan.github.io/jq/manual/#Variable func variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) { log.Debug("variable loop!") results := list.New() var evaluateAllTogether = true for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether if !evaluateAllTogether { break } } if evaluateAllTogether { return variableLoopSingleChild(d, context, originalExp) } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { result, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp) if err != nil { return Context{}, err } results.PushBackList(result.MatchingNodes) } return context.ChildContext(results), nil } func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) { variableExp := originalExp.LHS lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS) if err != nil { return Context{}, err } if variableExp.RHS.Operation.OperationType.Type != "GET_VARIABLE" { return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo") } variableName := variableExp.RHS.Operation.StringValue prefs := variableExp.Operation.Preferences.(assignVarPreferences) results := list.New() // now we loop over lhs, set variable to each result and calculate originalExp.Rhs for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode))) var variableValue = list.New() if prefs.IsReference { variableValue.PushBack(el.Value) } else { candidateCopy := el.Value.(*CandidateNode).Copy() variableValue.PushBack(candidateCopy) } newContext := context.ChildContext(context.MatchingNodes) newContext.SetVariable(variableName, variableValue) rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS) if err != nil { return Context{}, err } log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len()) results.PushBackList(rhs.MatchingNodes) } // if there is no LHS - then I guess we just calculate originalExp.Rhs if lhs.MatchingNodes.Len() == 0 { return d.GetMatchingNodes(context, originalExp.RHS) } return context.ChildContext(results), nil } ================================================ FILE: pkg/yqlib/operator_variables_test.go ================================================ package yqlib import ( "testing" ) var variableOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{}`, expression: `.a.b as $foo | .`, expected: []string{ "D0, P[], (!!map)::{}\n", }, }, { skipDoc: true, document: `{}`, expression: `.a.b as $foo`, expectedError: "must use variable with a pipe, e.g. `exp as $x | ...`", }, { document: "a: [cat]", skipDoc: true, expression: "(.[] | {.name: .}) as $item | .", expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`, }, { description: "Single value variable", document: `a: cat`, expression: `.a as $foo | $foo`, expected: []string{ "D0, P[a], (!!str)::cat\n", }, }, { description: "Multi value variable", document: `[cat, dog]`, expression: `.[] as $foo | $foo`, expected: []string{ "D0, P[0], (!!str)::cat\n", "D0, P[1], (!!str)::dog\n", }, }, { skipDoc: true, document: `[1, 2]`, expression: `.[] | . as $f | select($f == 2)`, expected: []string{ "D0, P[1], (!!int)::2\n", }, }, { skipDoc: true, document: `[1, 2]`, expression: `[.[] | . as $f | $f + 1]`, expected: []string{ "D0, P[], (!!seq)::- 2\n- 3\n", }, }, { description: "Using variables as a lookup", subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)", document: `{"posts": [{"title": "First post", "author": "anon"}, {"title": "A well-written article", "author": "person1"}], "realnames": {"anon": "Anonymous Coward", "person1": "Person McPherson"}}`, expression: `.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}`, expected: []string{ "D0, P[], (!!map)::title: \"First post\"\nauthor: \"Anonymous Coward\"\n", "D0, P[], (!!map)::title: \"A well-written article\"\nauthor: \"Person McPherson\"\n", }, }, { description: "Using variables to swap values", document: "a: a_value\nb: b_value", expression: `.a as $x | .b as $y | .b = $x | .a = $y`, expected: []string{ "D0, P[], (!!map)::a: b_value\nb: a_value\n", }, }, { description: "Use ref to reference a path repeatedly", subdescription: "Note: You may find the `with` operator more useful.", document: `a: {b: thing, c: something}`, expression: `.a.b ref $x | $x = "new" | $x style="double"`, expected: []string{ "D0, P[], (!!map)::a: {b: \"new\", c: something}\n", }, }, } func TestVariableOperatorScenarios(t *testing.T) { for _, tt := range variableOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "variable-operators", variableOperatorScenarios) } ================================================ FILE: pkg/yqlib/operator_with.go ================================================ package yqlib import "fmt" func withOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("withOperator") // with(path, exp) if expressionNode.RHS.Operation.OperationType != blockOpType { return Context{}, fmt.Errorf("with must be given a block (;), got %v instead", expressionNode.RHS.Operation.OperationType.Type) } pathExp := expressionNode.RHS.LHS updateContext, err := d.GetMatchingNodes(context, pathExp) if err != nil { return Context{}, err } updateExp := expressionNode.RHS.RHS for el := updateContext.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) _, err = d.GetMatchingNodes(updateContext.SingleChildContext(candidate), updateExp) if err != nil { return Context{}, err } } return context, nil } ================================================ FILE: pkg/yqlib/operator_with_test.go ================================================ package yqlib import "testing" var withOperatorScenarios = []expressionScenario{ { description: "Update and style", document: `a: {deeply: {nested: value}}`, expression: `with(.a.deeply.nested; . = "newValue" | . style="single")`, expected: []string{ "D0, P[], (!!map)::a: {deeply: {nested: 'newValue'}}\n", }, }, { description: "with splat", skipDoc: true, document: `a: {deeply: {nested: value}}`, expression: `with(.a.deeply.nested; . = "newValue" | . style="single")[]`, expected: []string{ "D0, P[a], (!!map)::{deeply: {nested: 'newValue'}}\n", }, }, { description: "Update multiple deeply nested properties", document: `a: {deeply: {nested: value, other: thing}}`, expression: `with(.a.deeply; .nested = "newValue" | .other= "newThing")`, expected: []string{ "D0, P[], (!!map)::a: {deeply: {nested: newValue, other: newThing}}\n", }, }, { description: "Update array elements relatively", subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.", document: `myArray: [{a: apple},{a: banana}]`, expression: `with(.myArray[]; .b = .a + " yum")`, expected: []string{ "D0, P[], (!!map)::myArray: [{a: apple, b: apple yum}, {a: banana, b: banana yum}]\n", }, }, { description: "Update array elements relatively +=", skipDoc: true, subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.", document: `myArray: [{a: apple},{a: banana}]`, expression: `with(.myArray[]; .a += .a)`, expected: []string{ "D0, P[], (!!map)::myArray: [{a: appleapple}, {a: bananabanana}]\n", }, }, } func TestWithOperatorScenarios(t *testing.T) { for _, tt := range withOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "with", withOperatorScenarios) } ================================================ FILE: pkg/yqlib/operators.go ================================================ package yqlib import ( "container/list" "fmt" "github.com/jinzhu/copier" logging "gopkg.in/op/go-logging.v1" ) type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) type compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) { lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } // tricky logic when we are running *= with flags. // we have an op like: .a *=nc .b // which should roughly translate to .a =c .a *nc .b // note that the 'n' flag only applies to the multiple op, not the assignment // but the clobber flag applies to both! prefs := assignPreferences{} switch typedPref := expressionNode.Operation.Preferences.(type) { case assignPreferences: prefs = typedPref case multiplyPreferences: prefs.ClobberCustomTags = typedPref.AssignPrefs.ClobberCustomTags } assignmentOp := &Operation{OperationType: assignOpType, Preferences: prefs} for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) clone := candidate.Copy() valueCopyExp := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: clone}} valueExpression := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: candidate}} assignmentOpNode := &ExpressionNode{Operation: assignmentOp, LHS: valueExpression, RHS: calculation(valueCopyExp, expressionNode.RHS)} _, err = d.GetMatchingNodes(context, assignmentOpNode) if err != nil { return Context{}, err } } return context, nil } func emptyOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) { context.MatchingNodes = list.New() return context, nil } type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) func resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, prefs crossFunctionPreferences, rhsExp *ExpressionNode, results *list.List) error { if prefs.LhsResultValue != nil { result, err := prefs.LhsResultValue(lhsCandidate) if err != nil { return err } else if result != nil { results.PushBack(result) return nil } } rhs, err := d.GetMatchingNodes(context, rhsExp) if err != nil { return err } if prefs.CalcWhenEmpty && rhs.MatchingNodes.Len() == 0 { resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, nil) if err != nil { return err } if resultCandidate != nil { results.PushBack(resultCandidate) } return nil } for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() { rhsCandidate := rightEl.Value.(*CandidateNode) if !log.IsEnabledFor(logging.DEBUG) { log.Debugf("Applying lhs: %v, rhsCandidate, %v", NodeToString(lhsCandidate), NodeToString(rhsCandidate)) } resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate) if err != nil { return err } if resultCandidate != nil { results.PushBack(resultCandidate) } } return nil } type crossFunctionPreferences struct { CalcWhenEmpty bool // if this returns a result node, // we wont bother calculating the RHS LhsResultValue func(*CandidateNode) (*CandidateNode, error) Calculation crossFunctionCalculation } func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) { var results = list.New() lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) if err != nil { return Context{}, err } log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len()) if prefs.CalcWhenEmpty && context.MatchingNodes.Len() > 0 && lhs.MatchingNodes.Len() == 0 { err := resultsForRHS(d, context, nil, prefs, expressionNode.RHS, results) if err != nil { return Context{}, err } } for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { lhsCandidate := el.Value.(*CandidateNode) err = resultsForRHS(d, context, lhsCandidate, prefs, expressionNode.RHS, results) if err != nil { return Context{}, err } } return context.ChildContext(results), nil } func crossFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation crossFunctionCalculation, calcWhenEmpty bool) (Context, error) { prefs := crossFunctionPreferences{CalcWhenEmpty: calcWhenEmpty, Calculation: calculation} return crossFunctionWithPrefs(d, context, expressionNode, prefs) } func crossFunctionWithPrefs(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, prefs crossFunctionPreferences) (Context, error) { var results = list.New() var evaluateAllTogether = true for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether if !evaluateAllTogether { break } } if evaluateAllTogether { log.Debug("crossFunction evaluateAllTogether!") return doCrossFunc(d, context, expressionNode, prefs) } log.Debug("crossFunction evaluate apart!") for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, prefs) if err != nil { return Context{}, err } results.PushBackList(innerResults.MatchingNodes) } return context.ChildContext(results), nil } func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode { valString := "true" if !value { valString = "false" } noob := owner.CreateReplacement(ScalarNode, "!!bool", valString) if owner.IsMapKey { noob.IsMapKey = false noob.Key = owner } return noob } func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode { if len(path) == 0 { return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} } else if len(path) == 1 { lastPrefs := traversePrefs if targetKey { err := copier.Copy(&lastPrefs, traversePrefs) if err != nil { panic(err) } lastPrefs.IncludeMapKeys = true lastPrefs.DontIncludeMapValues = true } return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} } return &ExpressionNode{ Operation: &Operation{OperationType: shortPipeOpType}, LHS: createTraversalTree(path[0:1], traversePrefs, false), RHS: createTraversalTree(path[1:], traversePrefs, targetKey), } } ================================================ FILE: pkg/yqlib/operators_compare_test.go ================================================ package yqlib import ( "testing" ) var compareOperatorScenarios = []expressionScenario{ // ints, not equal { description: "Compare numbers (>)", document: "a: 5\nb: 4", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, expression: "(.k | length) >= 0", expected: []string{ "D0, P[k], (!!bool)::true\n", }, }, { skipDoc: true, expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`, expected: []string{ "D0, P[], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5\nb: 4", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, description: "Compare integers (>=)", document: "a: 5\nb: 4", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5\nb: 4", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, // ints, equal { skipDoc: true, description: "Compare equal numbers (>)", document: "a: 5\nb: 5", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 5\nb: 5", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { description: "Compare equal numbers (>=)", document: "a: 5\nb: 5", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5\nb: 5", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, // floats, not equal { skipDoc: true, document: "a: 5.2\nb: 4.1", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5.2\nb: 4.1", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 5.2\nb: 4.1", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5.5\nb: 4.1", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, // floats, equal { skipDoc: true, document: "a: 5.5\nb: 5.5", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 5.5\nb: 5.5", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 5.1\nb: 5.1", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 5.1\nb: 5.1", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, // strings, not equal { description: "Compare strings", subdescription: "Compares strings by their bytecode.", document: "a: zoo\nb: apple", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: zoo\nb: apple", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: zoo\nb: apple", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: zoo\nb: apple", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, // strings, equal { skipDoc: true, document: "a: cat\nb: cat", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: cat\nb: cat", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: cat\nb: cat", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: cat\nb: cat", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, // datetime, not equal { description: "Compare date times", subdescription: "You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, // datetime, equal { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, document: "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z", expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, // both null { description: "Both sides are null: > is false", expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { description: "Both sides are null: >= is true", expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, { skipDoc: true, expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::true\n", }, }, // one null { skipDoc: true, description: "One side is null: > is false", document: `a: 5`, expression: ".a > .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: `a: 5`, expression: ".a < .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, description: "One side is null: >= is false", document: `a: 5`, expression: ".a >= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: `a: 5`, expression: ".a <= .b", expected: []string{ "D0, P[a], (!!bool)::false\n", }, }, { skipDoc: true, document: `a: 5`, expression: ".b <= .a", expected: []string{ "D0, P[b], (!!bool)::false\n", }, }, { skipDoc: true, document: `a: 5`, expression: ".b < .a", expected: []string{ "D0, P[b], (!!bool)::false\n", }, }, } func TestCompareOperatorScenarios(t *testing.T) { for _, tt := range compareOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "compare", compareOperatorScenarios) } var minOperatorScenarios = []expressionScenario{ { description: "Minimum int", document: "[99, 16, 12, 6, 66]\n", expression: `min`, expected: []string{ "D0, P[3], (!!int)::6\n", }, }, { description: "Minimum string", document: "[foo, bar, baz]\n", expression: `min`, expected: []string{ "D0, P[1], (!!str)::bar\n", }, }, { description: "Minimum of empty", document: "[]\n", expression: `min`, expected: []string{}, }, } func TestMinOperatorScenarios(t *testing.T) { for _, tt := range minOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "min", minOperatorScenarios) } var maxOperatorScenarios = []expressionScenario{ { description: "Maximum int", document: "[99, 16, 12, 6, 66]\n", expression: `max`, expected: []string{ "D0, P[0], (!!int)::99\n", }, }, { description: "Maximum string", document: "[foo, bar, baz]\n", expression: `max`, expected: []string{ "D0, P[0], (!!str)::foo\n", }, }, { description: "Maximum of empty", document: "[]\n", expression: `max`, expected: []string{}, }, } func TestMaxOperatorScenarios(t *testing.T) { for _, tt := range maxOperatorScenarios { testScenario(t, &tt) } documentOperatorScenarios(t, "max", maxOperatorScenarios) } ================================================ FILE: pkg/yqlib/operators_test.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "fmt" "io" "io/fs" "os" "sort" "strings" "testing" "time" "github.com/mikefarah/yq/v4/test" logging "gopkg.in/op/go-logging.v1" ) type expressionScenario struct { description string subdescription string explanation []string environmentVariables map[string]string document string document2 string expression string expected []string skipDoc bool expectedError string dontFormatInputForDoc bool // dont format input doc for documentation generation requiresFormat string skipForGoccy bool } var goccyTesting = false var testingDecoder = NewYamlDecoder(ConfiguredYamlPreferences) func TestMain(m *testing.M) { logging.SetLevel(logging.WARNING, "") if os.Getenv("DEBUG") == "true" { logging.SetLevel(logging.DEBUG, "") } ConfiguredYamlPreferences.ColorsEnabled = false ConfiguredJSONPreferences.ColorsEnabled = false goccyTesting = os.Getenv("GOCCY") == "true" if goccyTesting { testingDecoder = NewGoccyYAMLDecoder() } Now = func() time.Time { return time.Date(2021, time.May, 19, 1, 2, 3, 4, time.UTC) } code := m.Run() os.Exit(code) } func NewSimpleYamlPrinter(writer io.Writer, unwrapScalar bool, indent int, printDocSeparators bool) Printer { prefs := ConfiguredYamlPreferences.Copy() prefs.PrintDocSeparators = printDocSeparators prefs.UnwrapScalar = unwrapScalar prefs.Indent = indent return NewPrinter(NewYamlEncoder(prefs), NewSinglePrinterWriter(writer)) } func readDocument(content string, fakefilename string, fakeFileIndex int) (*list.List, error) { reader := bufio.NewReader(strings.NewReader(content)) return readDocuments(reader, fakefilename, fakeFileIndex, testingDecoder) } func testScenario(t *testing.T, s *expressionScenario) { if s.skipForGoccy { return } log.Debugf("\n\ntesting scenario %v", s.description) var err error node, err := getExpressionParser().ParseExpression(s.expression) if err != nil { t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) return } inputs := list.New() if s.document != "" { inputs, err = readDocument(s.document, "sample.yml", 0) if err != nil { t.Error(err, s.document, s.expression) return } if s.document2 != "" { moreInputs, err := readDocument(s.document2, "another.yml", 1) if err != nil { t.Error(err, s.document2, s.expression) return } inputs.PushBackList(moreInputs) } } else { candidateNode := &CandidateNode{ document: 0, filename: "", Tag: "!!null", Kind: ScalarNode, fileIndex: 0, } inputs.PushBack(candidateNode) } for name, value := range s.environmentVariables { os.Setenv(name, value) } context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node) if s.expectedError != "" { if err == nil { t.Errorf("Expected error '%v' but it worked!", s.expectedError) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) } return } if s.requiresFormat != "" { format := s.requiresFormat inputFormat, err := FormatFromString(format) if err != nil { t.Error(err) } if decoder := inputFormat.DecoderFactory(); decoder == nil { t.Skipf("no support for %s input format", format) } outputFormat, err := FormatFromString(format) if err != nil { t.Error(err) } if encoder := configureEncoder(outputFormat, 4); encoder == nil { t.Skipf("no support for %s output format", format) } } if err != nil { t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression)) return } test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) } func resultToString(t *testing.T, n *CandidateNode) string { var valueBuffer bytes.Buffer log.Debugf("printing result %v", NodeToString(n)) printer := NewSimpleYamlPrinter(bufio.NewWriter(&valueBuffer), true, 4, true) err := printer.PrintResults(n.AsList()) if err != nil { t.Error(err) return "" } tag := n.Tag if n.Kind == AliasNode { tag = "alias" } return fmt.Sprintf(`D%v, P%v, (%v)::%v`, n.GetDocument(), n.GetPath(), tag, valueBuffer.String()) } func resultsToString(t *testing.T, results *list.List) []string { var pretty = make([]string, 0) for el := results.Front(); el != nil; el = el.Next() { n := el.Value.(*CandidateNode) output := resultToString(t, n) pretty = append(pretty, output) } return pretty } func writeOrPanic(w *bufio.Writer, text string) { _, err := w.WriteString(text) if err != nil { panic(err) } } func copySnippet(source string, out *os.File) error { _, err := os.Stat(source) if os.IsNotExist(err) { return nil } in, err := os.Open(source) if err != nil { return err } defer safelyCloseFile(in) _, err = io.Copy(out, in) return err } func formatYaml(yaml string, filename string) string { var output bytes.Buffer printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true) node, err := getExpressionParser().ParseExpression(".. style= \"\"") if err != nil { panic(err) } streamEvaluator := NewStreamEvaluator() _, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, testingDecoder) if err != nil { panic(err) } return output.String() } type documentScenarioFunc func(t *testing.T, writer *bufio.Writer, scenario interface{}) func documentScenarios(t *testing.T, folder string, title string, scenarios []interface{}, documentScenario documentScenarioFunc) { filename := fmt.Sprintf("doc/%v/%v.md", folder, title) f, err := os.Create(filename) if err != nil { t.Error(err) return } defer f.Close() source := fmt.Sprintf("doc/%v/headers/%v.md", folder, title) err = copySnippet(source, f) if err != nil { t.Error(err) return } err = copySnippet("doc/notification-snippet.md", f) if err != nil { t.Error(err) return } w := bufio.NewWriter(f) writeOrPanic(w, "\n") for _, s := range scenarios { documentScenario(t, w, s) } w.Flush() } func appendOperatorDocumentScenario(t *testing.T, title string, scenarios []expressionScenario) { filename := fmt.Sprintf("doc/%v/%v.md", "operators", title) f, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR|os.O_APPEND, fs.ModeAppend) if err != nil { t.Error(err) return } defer f.Close() w := bufio.NewWriter(f) for _, s := range scenarios { documentOperatorScenario(t, w, s) } w.Flush() } func documentOperatorScenarios(t *testing.T, title string, scenarios []expressionScenario) { genericScenarios := make([]interface{}, len(scenarios)) for i, s := range scenarios { genericScenarios[i] = s } documentScenarios(t, "operators", title, genericScenarios, documentOperatorScenario) } func documentOperatorScenario(t *testing.T, w *bufio.Writer, i interface{}) { s := i.(expressionScenario) if s.skipDoc { return } writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } formattedDoc, formattedDoc2 := documentInput(w, s) writeOrPanic(w, "will output\n") documentOutput(t, w, s, formattedDoc, formattedDoc2) if len(s.explanation) > 0 { writeOrPanic(w, "### Explanation:\n") for _, text := range s.explanation { writeOrPanic(w, fmt.Sprintf("- %v\n", text)) } writeOrPanic(w, "\n") } } func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { formattedDoc := "" formattedDoc2 := "" command := "" envCommand := "" envKeys := make([]string, 0, len(s.environmentVariables)) for k := range s.environmentVariables { envKeys = append(envKeys, k) } sort.Strings(envKeys) for _, name := range envKeys { value := s.environmentVariables[name] if envCommand == "" { envCommand = fmt.Sprintf("%v=\"%v\" ", name, value) } else { envCommand = fmt.Sprintf("%v %v=\"%v\" ", envCommand, name, value) } os.Setenv(name, value) } if s.document != "" { if s.dontFormatInputForDoc { formattedDoc = s.document + "\n" } else { formattedDoc = formatYaml(s.document, "sample.yml") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc)) files := "sample.yml" if s.document2 != "" { if s.dontFormatInputForDoc { formattedDoc2 = s.document2 + "\n" } else { formattedDoc2 = formatYaml(s.document2, "another.yml") } writeOrPanic(w, "And another sample another.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n", formattedDoc2)) files = "sample.yml another.yml" command = "eval-all " } writeOrPanic(w, "then\n") if s.expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) } else { writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) } } else { writeOrPanic(w, "Running\n") writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) } return formattedDoc, formattedDoc2 } func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) { var output bytes.Buffer var err error printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), true, 2, true) node, err := getExpressionParser().ParseExpression(s.expression) if err != nil { t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) return } inputs := list.New() if s.document != "" { inputs, err = readDocument(formattedDoc, "sample.yml", 0) if err != nil { t.Error(err, formattedDoc, "exp: "+s.expression) return } if s.document2 != "" { moreInputs, err := readDocument(formattedDoc2, "another.yml", 1) if err != nil { t.Error(err, formattedDoc2, "exp: "+s.expression) return } inputs.PushBackList(moreInputs) } } else { candidateNode := &CandidateNode{ document: 0, filename: "", Tag: "!!null", Kind: ScalarNode, fileIndex: 0, } inputs.PushBack(candidateNode) } context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node) if s.expectedError != "" && err != nil { writeOrPanic(w, fmt.Sprintf("```bash\nError: %v\n```\n\n", err.Error())) return } else if err != nil { t.Error(err, s.expression) return } err = printer.PrintResults(context.MatchingNodes) if err != nil { t.Error(err, s.expression) } writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", output.String())) } ================================================ FILE: pkg/yqlib/printer.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "fmt" "io" "regexp" ) type Printer interface { PrintResults(matchingNodes *list.List) error PrintedAnything() bool //e.g. when given a front-matter doc, like jekyll SetAppendix(reader io.Reader) SetNulSepOutput(nulSepOutput bool) } type resultsPrinter struct { encoder Encoder printerWriter PrinterWriter firstTimePrinting bool previousDocIndex uint previousFileIndex int printedMatches bool treeNavigator DataTreeNavigator appendixReader io.Reader nulSepOutput bool } func NewPrinter(encoder Encoder, printerWriter PrinterWriter) Printer { return &resultsPrinter{ encoder: encoder, printerWriter: printerWriter, firstTimePrinting: true, treeNavigator: NewDataTreeNavigator(), nulSepOutput: false, } } func (p *resultsPrinter) SetNulSepOutput(nulSepOutput bool) { log.Debug("Setting NUL separator output") p.nulSepOutput = nulSepOutput } func (p *resultsPrinter) SetAppendix(reader io.Reader) { p.appendixReader = reader } func (p *resultsPrinter) PrintedAnything() bool { return p.printedMatches } func (p *resultsPrinter) printNode(node *CandidateNode, writer io.Writer) error { p.printedMatches = p.printedMatches || (node.Tag != "!!null" && (node.Tag != "!!bool" || node.Value != "false")) return p.encoder.Encode(writer, node) } func removeLastEOL(b *bytes.Buffer) { data := b.Bytes() n := len(data) if n >= 2 && data[n-2] == '\r' && data[n-1] == '\n' { b.Truncate(n - 2) } else if n >= 1 && (data[n-1] == '\r' || data[n-1] == '\n') { b.Truncate(n - 1) } } func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { log.Debug("PrintResults for %v matches", matchingNodes.Len()) if matchingNodes.Len() == 0 { log.Debug("no matching results, nothing to print") return nil } if !p.encoder.CanHandleAliases() { explodeOp := Operation{OperationType: explodeOpType} explodeNode := ExpressionNode{Operation: &explodeOp} context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode) if err != nil { return err } matchingNodes = context.MatchingNodes } if p.firstTimePrinting { node := matchingNodes.Front().Value.(*CandidateNode) p.previousDocIndex = node.GetDocument() p.previousFileIndex = node.GetFileIndex() p.firstTimePrinting = false } for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) log.Debug("print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex) log.Debug("%v", NodeToString(mappedDoc)) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) if errorWriting != nil { return errorWriting } commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeparator\$`) commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) if (p.previousDocIndex != mappedDoc.GetDocument() || p.previousFileIndex != mappedDoc.GetFileIndex()) && !commentStartsWithSeparator { if err := p.encoder.PrintDocumentSeparator(writer); err != nil { return err } } var destination io.Writer = writer tempBuffer := bytes.NewBuffer(nil) if p.nulSepOutput { destination = tempBuffer } if err := p.encoder.PrintLeadingContent(destination, mappedDoc.LeadingContent); err != nil { return err } if err := p.printNode(mappedDoc, destination); err != nil { return err } if p.nulSepOutput { removeLastEOL(tempBuffer) tempBufferBytes := tempBuffer.Bytes() if bytes.IndexByte(tempBufferBytes, 0) != -1 { return fmt.Errorf( "can't serialise value because it contains NUL char and you are using NUL separated output", ) } if _, err := writer.Write(tempBufferBytes); err != nil { return err } if _, err := writer.Write([]byte{0}); err != nil { return err } } p.previousDocIndex = mappedDoc.GetDocument() if err := writer.Flush(); err != nil { return err } log.Debugf("done printing results") } // what happens if I remove output format check? if p.appendixReader != nil { writer, err := p.printerWriter.GetWriter(nil) if err != nil { return err } log.Debug("Piping appendix reader...") betterReader := bufio.NewReader(p.appendixReader) _, err = io.Copy(writer, betterReader) if err != nil { return err } if err := writer.Flush(); err != nil { return err } } return nil } ================================================ FILE: pkg/yqlib/printer_node_info.go ================================================ package yqlib import ( "bufio" "container/list" "io" "go.yaml.in/yaml/v4" ) type nodeInfoPrinter struct { printerWriter PrinterWriter appendixReader io.Reader printedMatches bool } func NewNodeInfoPrinter(printerWriter PrinterWriter) Printer { return &nodeInfoPrinter{ printerWriter: printerWriter, } } func (p *nodeInfoPrinter) SetNulSepOutput(_ bool) { } func (p *nodeInfoPrinter) SetAppendix(reader io.Reader) { p.appendixReader = reader } func (p *nodeInfoPrinter) PrintedAnything() bool { return p.printedMatches } func (p *nodeInfoPrinter) PrintResults(matchingNodes *list.List) error { for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) writer, errorWriting := p.printerWriter.GetWriter(mappedDoc) if errorWriting != nil { return errorWriting } bytes, err := yaml.Marshal(mappedDoc.ConvertToNodeInfo()) if err != nil { return err } if _, err := writer.Write(bytes); err != nil { return err } if _, err := writer.Write([]byte("\n")); err != nil { return err } p.printedMatches = true if err := writer.Flush(); err != nil { return err } } if p.appendixReader != nil { writer, err := p.printerWriter.GetWriter(nil) if err != nil { return err } log.Debug("Piping appendix reader...") betterReader := bufio.NewReader(p.appendixReader) _, err = io.Copy(writer, betterReader) if err != nil { return err } if err := writer.Flush(); err != nil { return err } } return nil } ================================================ FILE: pkg/yqlib/printer_node_info_test.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) func TestNodeInfoPrinter_PrintResults(t *testing.T) { // Create a simple CandidateNode node := &CandidateNode{ Kind: ScalarNode, Style: DoubleQuotedStyle, Tag: "!!str", Value: "hello world", Line: 5, Column: 7, HeadComment: "head", LineComment: "line", FootComment: "foot", Anchor: "anchor", } listNodes := list.New() listNodes.PushBack(node) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } outStr := output.String() // Check for key NodeInfo fields in YAML output using substring checks test.AssertResult(t, true, strings.Contains(outStr, "kind: ScalarNode")) test.AssertResult(t, true, strings.Contains(outStr, "style: DoubleQuotedStyle")) test.AssertResult(t, true, strings.Contains(outStr, "tag: '!!str'")) test.AssertResult(t, true, strings.Contains(outStr, "value: hello world")) test.AssertResult(t, true, strings.Contains(outStr, "line: 5")) test.AssertResult(t, true, strings.Contains(outStr, "column: 7")) test.AssertResult(t, true, strings.Contains(outStr, "headComment: head")) test.AssertResult(t, true, strings.Contains(outStr, "lineComment: line")) test.AssertResult(t, true, strings.Contains(outStr, "footComment: foot")) test.AssertResult(t, true, strings.Contains(outStr, "anchor: anchor")) } func TestNodeInfoPrinter_PrintedAnything_True(t *testing.T) { node := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "test", } listNodes := list.New() listNodes.PushBack(node) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) // Before printing, should be false test.AssertResult(t, false, printer.PrintedAnything()) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } // After printing, should be true test.AssertResult(t, true, printer.PrintedAnything()) } func TestNodeInfoPrinter_PrintedAnything_False(t *testing.T) { listNodes := list.New() var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } // No nodes printed, should still be false test.AssertResult(t, false, printer.PrintedAnything()) } func TestNodeInfoPrinter_SetNulSepOutput(_ *testing.T) { var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) // Should not panic or error printer.SetNulSepOutput(true) printer.SetNulSepOutput(false) } func TestNodeInfoPrinter_SetAppendix(t *testing.T) { node := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "test", } listNodes := list.New() listNodes.PushBack(node) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) appendixText := "This is appendix text\n" appendixReader := strings.NewReader(appendixText) printer.SetAppendix(appendixReader) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } outStr := output.String() test.AssertResult(t, true, strings.Contains(outStr, "test")) test.AssertResult(t, true, strings.Contains(outStr, appendixText)) } func TestNodeInfoPrinter_MultipleNodes(t *testing.T) { node1 := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "first", } node2 := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "second", } listNodes := list.New() listNodes.PushBack(node1) listNodes.PushBack(node2) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } outStr := output.String() test.AssertResult(t, true, strings.Contains(outStr, "value: first")) test.AssertResult(t, true, strings.Contains(outStr, "value: second")) } func TestNodeInfoPrinter_SequenceNode(t *testing.T) { node := &CandidateNode{ Kind: SequenceNode, Tag: "!!seq", Style: FlowStyle, } listNodes := list.New() listNodes.PushBack(node) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } outStr := output.String() test.AssertResult(t, true, strings.Contains(outStr, "kind: SequenceNode")) test.AssertResult(t, true, strings.Contains(outStr, "tag: '!!seq'")) test.AssertResult(t, true, strings.Contains(outStr, "style: FlowStyle")) } func TestNodeInfoPrinter_MappingNode(t *testing.T) { node := &CandidateNode{ Kind: MappingNode, Tag: "!!map", } listNodes := list.New() listNodes.PushBack(node) var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } outStr := output.String() test.AssertResult(t, true, strings.Contains(outStr, "kind: MappingNode")) test.AssertResult(t, true, strings.Contains(outStr, "tag: '!!map'")) } func TestNodeInfoPrinter_EmptyList(t *testing.T) { listNodes := list.New() var output bytes.Buffer writer := bufio.NewWriter(&output) printer := NewNodeInfoPrinter(NewSinglePrinterWriter(writer)) err := printer.PrintResults(listNodes) writer.Flush() if err != nil { t.Fatalf("PrintResults error: %v", err) } test.AssertResult(t, "", output.String()) test.AssertResult(t, false, printer.PrintedAnything()) } ================================================ FILE: pkg/yqlib/printer_test.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "strings" "testing" "github.com/mikefarah/yq/v4/test" ) var multiDocSample = `a: banana --- a: apple --- a: coconut ` var multiDocSampleLeadingExpected = `# go cats --- a: banana --- a: apple --- # cool a: coconut ` func nodeToList(candidate *CandidateNode) *list.List { elMap := list.New() elMap.PushBack(candidate) return elMap } func TestPrinterMultipleDocsInSequenceOnly(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } el := inputs.Front() sample1 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() sample2 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() sample3 := nodeToList(el.Value.(*CandidateNode)) err = printer.PrintResults(sample1) if err != nil { panic(err) } err = printer.PrintResults(sample2) if err != nil { panic(err) } err = printer.PrintResults(sample3) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, multiDocSample, output.String()) } func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } el := inputs.Front() el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeparator$\n" sample1 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() el.Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n" sample2 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() el.Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n# cool\n" sample3 := nodeToList(el.Value.(*CandidateNode)) err = printer.PrintResults(sample1) if err != nil { panic(err) } err = printer.PrintResults(sample2) if err != nil { panic(err) } err = printer.PrintResults(sample3) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, multiDocSampleLeadingExpected, output.String()) } func TestPrinterMultipleFilesInSequence(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } el := inputs.Front() elNode := el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 0 sample1 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 1 sample2 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 2 sample3 := nodeToList(elNode) err = printer.PrintResults(sample1) if err != nil { panic(err) } err = printer.PrintResults(sample2) if err != nil { panic(err) } err = printer.PrintResults(sample3) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, multiDocSample, output.String()) } func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } el := inputs.Front() elNode := el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 0 elNode.LeadingContent = "# go cats\n$yqDocSeparator$\n" sample1 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 1 elNode.LeadingContent = "$yqDocSeparator$\n" sample2 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.document = 0 elNode.fileIndex = 2 elNode.LeadingContent = "$yqDocSeparator$\n# cool\n" sample3 := nodeToList(elNode) err = printer.PrintResults(sample1) if err != nil { panic(err) } err = printer.PrintResults(sample2) if err != nil { panic(err) } err = printer.PrintResults(sample3) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, multiDocSampleLeadingExpected, output.String()) } func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } err = printer.PrintResults(inputs) if err != nil { panic(err) } writer.Flush() test.AssertResult(t, multiDocSample, output.String()) } func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeparator$\n" err = printer.PrintResults(inputs) if err != nil { panic(err) } writer.Flush() expected := `# go cats --- a: banana --- a: apple --- a: coconut ` test.AssertResult(t, expected, output.String()) } func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n" err = printer.PrintResults(inputs) if err != nil { panic(err) } writer.Flush() expected := `--- a: banana --- a: apple --- a: coconut ` test.AssertResult(t, expected, output.String()) } func TestPrinterScalarWithLeadingCont(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) node, err := getExpressionParser().ParseExpression(".a") if err != nil { panic(err) } streamEvaluator := NewStreamEvaluator() _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } writer.Flush() expected := `banana --- apple --- coconut ` test.AssertResult(t, expected, output.String()) } func TestPrinterMultipleDocsJson(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators // when outputting JSON. prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = 0 encoder := NewJSONEncoder(prefs) if encoder == nil { t.Skipf("no support for %s output format", "json") } printer := NewPrinter(encoder, NewSinglePrinterWriter(writer)) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } inputs.Front().Value.(*CandidateNode).LeadingContent = "# ignore this\n" err = printer.PrintResults(inputs) if err != nil { panic(err) } expected := `{"a":"banana"} {"a":"apple"} {"a":"coconut"} ` writer.Flush() test.AssertResult(t, expected, output.String()) } func TestPrinterNulSeparator(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, false) printer.SetNulSepOutput(true) node, err := getExpressionParser().ParseExpression(".a") if err != nil { panic(err) } streamEvaluator := NewStreamEvaluator() _, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } writer.Flush() expected := "banana\x00apple\x00coconut\x00" test.AssertResult(t, expected, output.String()) } func TestPrinterNulSeparatorWithJson(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators // when outputting JSON. prefs := ConfiguredJSONPreferences.Copy() prefs.Indent = 0 encoder := NewJSONEncoder(prefs) if encoder == nil { t.Skipf("no support for %s output format", "json") } printer := NewPrinter(encoder, NewSinglePrinterWriter(writer)) printer.SetNulSepOutput(true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } inputs.Front().Value.(*CandidateNode).LeadingContent = "# ignore this\n" err = printer.PrintResults(inputs) if err != nil { panic(err) } expected := `{"a":"banana"}` + "\x00" + `{"a":"apple"}` + "\x00" + `{"a":"coconut"}` + "\x00" writer.Flush() test.AssertResult(t, expected, output.String()) } func TestPrinterRootUnwrap(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, false) node, err := getExpressionParser().ParseExpression(".") if err != nil { panic(err) } streamEvaluator := NewStreamEvaluator() _, err = streamEvaluator.Evaluate("sample", strings.NewReader("'a'"), node, printer, NewYamlDecoder(ConfiguredYamlPreferences)) if err != nil { panic(err) } writer.Flush() expected := `a ` test.AssertResult(t, expected, output.String()) } func TestRemoveLastEOL(t *testing.T) { // Test with \r\n buffer := bytes.NewBufferString("test\r\n") removeLastEOL(buffer) test.AssertResult(t, "test", buffer.String()) // Test with \n only buffer = bytes.NewBufferString("test\n") removeLastEOL(buffer) test.AssertResult(t, "test", buffer.String()) // Test with \r only buffer = bytes.NewBufferString("test\r") removeLastEOL(buffer) test.AssertResult(t, "test", buffer.String()) // Test with no EOL buffer = bytes.NewBufferString("test") removeLastEOL(buffer) test.AssertResult(t, "test", buffer.String()) // Test with empty buffer buffer = bytes.NewBufferString("") removeLastEOL(buffer) test.AssertResult(t, "", buffer.String()) // Test with multiple \r\n buffer = bytes.NewBufferString("line1\r\nline2\r\n") removeLastEOL(buffer) test.AssertResult(t, "line1\r\nline2", buffer.String()) } func TestPrinterPrintedAnything(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) test.AssertResult(t, false, printer.PrintedAnything()) // Print a scalar value node := createStringScalarNode("test") nodeList := nodeToList(node) err := printer.PrintResults(nodeList) if err != nil { t.Fatal(err) } // Should now be true test.AssertResult(t, true, printer.PrintedAnything()) } func TestPrinterNulSeparatorWithNullChar(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, false) printer.SetNulSepOutput(true) // Create a node with null character node := createStringScalarNode("test\x00value") nodeList := nodeToList(node) err := printer.PrintResults(nodeList) if err == nil { t.Fatal("Expected error for null character in NUL separated output") } expectedError := "can't serialise value because it contains NUL char and you are using NUL separated output" if err.Error() != expectedError { t.Fatalf("Expected error '%s', got '%s'", expectedError, err.Error()) } } func TestPrinterSetNulSepOutput(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, false) // Test setting NUL separator output printer.SetNulSepOutput(true) test.AssertResult(t, true, true) // Placeholder assertion printer.SetNulSepOutput(false) // Should also not cause errors test.AssertResult(t, false, false) // Placeholder assertion } func TestPrinterSetAppendix(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, true, 2, true) // Test setting appendix appendix := strings.NewReader("appendix content") printer.SetAppendix(appendix) test.AssertResult(t, true, true) // Placeholder assertion } ================================================ FILE: pkg/yqlib/printer_writer.go ================================================ package yqlib import ( "bufio" "fmt" "io" "os" "path/filepath" "regexp" ) type PrinterWriter interface { GetWriter(node *CandidateNode) (*bufio.Writer, error) } type singlePrinterWriter struct { bufferedWriter *bufio.Writer } func NewSinglePrinterWriter(writer io.Writer) PrinterWriter { return &singlePrinterWriter{ bufferedWriter: bufio.NewWriter(writer), } } func (sp *singlePrinterWriter) GetWriter(_ *CandidateNode) (*bufio.Writer, error) { return sp.bufferedWriter, nil } type multiPrintWriter struct { treeNavigator DataTreeNavigator nameExpression *ExpressionNode extension string index int } func NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWriter { extension := "yml" switch format { case JSONFormat: extension = "json" case PropertiesFormat: extension = "properties" } return &multiPrintWriter{ nameExpression: expression, extension: extension, treeNavigator: NewDataTreeNavigator(), index: 0, } } func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error) { name := "" indexVariableNode := CandidateNode{Kind: ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", sp.index)} context := Context{MatchingNodes: node.AsList()} context.SetVariable("index", indexVariableNode.AsList()) result, err := sp.treeNavigator.GetMatchingNodes(context, sp.nameExpression) if err != nil { return nil, err } if result.MatchingNodes.Len() > 0 { name = result.MatchingNodes.Front().Value.(*CandidateNode).Value } var extensionRegexp = regexp.MustCompile(`\.[a-zA-Z0-9]+$`) if !extensionRegexp.MatchString(name) { name = fmt.Sprintf("%v.%v", name, sp.extension) } err = os.MkdirAll(filepath.Dir(name), 0750) if err != nil { return nil, err } f, err := os.Create(name) if err != nil { return nil, err } sp.index = sp.index + 1 return bufio.NewWriter(f), nil } ================================================ FILE: pkg/yqlib/properties.go ================================================ package yqlib type PropertiesPreferences struct { UnwrapScalar bool KeyValueSeparator string UseArrayBrackets bool } func NewDefaultPropertiesPreferences() PropertiesPreferences { return PropertiesPreferences{ UnwrapScalar: true, KeyValueSeparator: " = ", UseArrayBrackets: false, } } func (p *PropertiesPreferences) Copy() PropertiesPreferences { return PropertiesPreferences{ UnwrapScalar: p.UnwrapScalar, KeyValueSeparator: p.KeyValueSeparator, UseArrayBrackets: p.UseArrayBrackets, } } var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences() ================================================ FILE: pkg/yqlib/properties_test.go ================================================ package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const propertiesWithCommentsOnMap = `this.thing = hi hi # important notes # about this value this.value = cool ` const expectedPropertiesWithCommentsOnMapProps = `this.thing = hi hi # important notes # about this value this.value = cool ` const expectedPropertiesWithCommentsOnMapYaml = `this: thing: hi hi # important notes # about this value value: cool ` const propertiesWithCommentInArray = ` this.array.0 = cat # important notes # about dogs this.array.1 = dog ` const expectedPropertiesWithCommentInArrayProps = `this.array.0 = cat # important notes # about dogs this.array.1 = dog ` const expectedPropertiesWithCommentInArrayYaml = `this: array: - cat # important notes # about dogs - dog ` const samplePropertiesYaml = `# block comments come through person: # neither do comments on maps name: Mike Wazowski # comments on values appear pets: - cat # comments on array values appear - nested: - list entry food: [pizza] # comments on arrays do not emptyArray: [] emptyMap: [] ` const expectedPropertiesUnwrapped = `# block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ` const expectedPropertiesUnwrappedArrayBrackets = `# block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets[0] = cat person.pets[1].nested[0] = list entry person.food[0] = pizza ` const expectedPropertiesUnwrappedCustomSeparator = `# block comments come through # comments on values appear person.name :@ Mike Wazowski # comments on array values appear person.pets.0 :@ cat person.pets.1.nested.0 :@ list entry person.food.0 :@ pizza ` const expectedPropertiesWrapped = `# block comments come through # comments on values appear person.name = "Mike Wazowski" # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = "list entry" person.food.0 = pizza ` const expectedUpdatedProperties = `# block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = dog person.pets.1.nested.0 = list entry person.food.0 = pizza ` const expectedDecodedYaml = `person: # block comments come through # comments on values appear name: Mike Wazowski pets: # comments on array values appear - cat - nested: - list entry food: - pizza ` const expectedDecodedPersonYaml = `# block comments come through # comments on values appear name: Mike Wazowski pets: # comments on array values appear - cat - nested: - list entry food: - pizza ` const expectedPropertiesNoComments = `person.name = Mike Wazowski person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza ` const expectedPropertiesWithEmptyMapsAndArrays = `# block comments come through # comments on values appear person.name = Mike Wazowski # comments on array values appear person.pets.0 = cat person.pets.1.nested.0 = list entry person.food.0 = pizza emptyArray = emptyMap = ` var propertyScenarios = []formatScenario{ { description: "Encode properties", subdescription: "Note that empty arrays and maps are not encoded by default.", input: samplePropertiesYaml, expected: expectedPropertiesUnwrapped, }, { description: "Encode properties with array brackets", subdescription: "Declare the --properties-array-brackets flag to give array paths in brackets (e.g. SpringBoot).", input: samplePropertiesYaml, expected: expectedPropertiesUnwrappedArrayBrackets, scenarioType: "encode-array-brackets", }, { description: "Encode properties - custom separator", subdescription: "Use the --properties-separator flag to specify your own key/value separator.", input: samplePropertiesYaml, expected: expectedPropertiesUnwrappedCustomSeparator, scenarioType: "encode-custom-separator", }, { description: "Encode properties: scalar encapsulation", subdescription: "Note that string values with blank characters in them are encapsulated with double quotes", input: samplePropertiesYaml, expected: expectedPropertiesWrapped, scenarioType: "encode-wrapped", }, { description: "Encode properties: no comments", input: samplePropertiesYaml, expected: expectedPropertiesNoComments, expression: `... comments = ""`, }, { description: "Encode properties: include empty maps and arrays", subdescription: "Use a yq expression to set the empty maps and sequences to your desired value.", expression: `(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) = ""`, input: samplePropertiesYaml, expected: expectedPropertiesWithEmptyMapsAndArrays, }, { description: "Decode properties", input: expectedPropertiesUnwrapped, expected: expectedDecodedYaml, scenarioType: "decode", }, { skipDoc: true, description: "Decode properties - keeps key information", input: expectedPropertiesUnwrapped, expression: ".person.name | key", expected: "name\n", scenarioType: "decode", }, { skipDoc: true, description: "Decode properties - keeps parent information", input: expectedPropertiesUnwrapped, expression: ".person.name | parent", expected: expectedDecodedPersonYaml, scenarioType: "decode", }, { skipDoc: true, description: "Decode properties - keeps path information", input: expectedPropertiesUnwrapped, expression: ".person.name | path", expected: "- person\n- name\n", scenarioType: "decode", }, { description: "Decode properties: numbers", subdescription: "All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.", input: "a.b = 10", expression: " (.. | select(tag == \"!!str\")) |= from_yaml", expected: "a:\n b: 10\n", scenarioType: "decode", }, { description: "Decode properties - array should be a map", subdescription: "If you have a numeric map key in your property files, use array_to_map to convert them to maps.", input: `things.10 = mike`, expression: `.things |= array_to_map`, expected: "things:\n 10: mike\n", scenarioType: "decode", }, { description: "does not expand automatically", skipDoc: true, input: "mike = ${dontExpand} this", expected: "mike: ${dontExpand} this\n", scenarioType: "decode", }, { description: "print scalar", skipDoc: true, input: "mike = cat", expression: ".mike", expected: "cat\n", scenarioType: "roundtrip", }, { description: "Roundtrip", input: expectedPropertiesUnwrapped, expression: `.person.pets.0 = "dog"`, expected: expectedUpdatedProperties, scenarioType: "roundtrip", }, { skipDoc: true, description: "comments on arrays roundtrip", input: propertiesWithCommentInArray, expected: expectedPropertiesWithCommentInArrayProps, scenarioType: "roundtrip", }, { skipDoc: true, description: "comments on arrays decode", input: propertiesWithCommentInArray, expected: expectedPropertiesWithCommentInArrayYaml, scenarioType: "decode", }, { skipDoc: true, description: "comments on map roundtrip", input: propertiesWithCommentsOnMap, expected: expectedPropertiesWithCommentsOnMapProps, scenarioType: "roundtrip", }, { skipDoc: true, description: "comments on map decode", input: propertiesWithCommentsOnMap, expected: expectedPropertiesWithCommentsOnMapYaml, scenarioType: "decode", }, { description: "Empty doc", skipDoc: true, input: "", expected: "", scenarioType: "decode", }, } func documentUnwrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression prefs := NewDefaultPropertiesPreferences() useArrayBracketsFlag := "" useCustomSeparatorFlag := "" switch s.scenarioType { case "encode-array-brackets": useArrayBracketsFlag = " --properties-array-brackets" prefs.UseArrayBrackets = true case "encode-custom-separator": prefs.KeyValueSeparator = " :@ " useCustomSeparatorFlag = ` --properties-separator=" :@ "` } if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props%v%v '%v' sample.yml\n```\n", useArrayBracketsFlag, useCustomSeparatorFlag, expression)) } else { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props%v%v sample.yml\n```\n", useArrayBracketsFlag, useCustomSeparatorFlag)) } writeOrPanic(w, "will output\n") prefs.UnwrapScalar = true writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)))) } func documentWrappedEncodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=props --unwrapScalar=false '%v' sample.yml\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -o=props --unwrapScalar=false sample.yml\n```\n") } writeOrPanic(w, "will output\n") prefs := ConfiguredPropertiesPreferences.Copy() prefs.UnwrapScalar = false writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)))) } func documentDecodePropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.properties file of:\n") writeOrPanic(w, fmt.Sprintf("```properties\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=props '%v' sample.properties\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -p=props sample.properties\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentRoundTripPropertyScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.properties file of:\n") writeOrPanic(w, fmt.Sprintf("```properties\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=props -o=props '%v' sample.properties\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -p=props -o=props sample.properties\n```\n") } writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```properties\n%v```\n\n", mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences)))) } func documentPropertyScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "encode-array-brackets", "encode-custom-separator": documentUnwrappedEncodePropertyScenario(w, s) case "decode": documentDecodePropertyScenario(w, s) case "encode-wrapped": documentWrappedEncodePropertyScenario(w, s) case "roundtrip": documentRoundTripPropertyScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func TestPropertyScenarios(t *testing.T) { for _, s := range propertyScenarios { switch s.scenarioType { case "": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description) case "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "encode-wrapped": prefs := ConfiguredPropertiesPreferences.Copy() prefs.UnwrapScalar = false test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description) case "encode-array-brackets": prefs := ConfiguredPropertiesPreferences.Copy() prefs.KeyValueSeparator = " = " prefs.UseArrayBrackets = true test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description) case "encode-custom-separator": prefs := ConfiguredPropertiesPreferences.Copy() prefs.KeyValueSeparator = " :@ " test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(prefs)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } genericScenarios := make([]interface{}, len(propertyScenarios)) for i, s := range propertyScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "properties", genericScenarios, documentPropertyScenario) } ================================================ FILE: pkg/yqlib/recipes_test.go ================================================ package yqlib import ( "testing" ) var bashEnvScript = `.[] |( ( select(kind == "scalar") | key + "='" + . + "'"), ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")") )` var nestedBashEnvScript = `.. |( ( select(kind == "scalar" and parent | kind != "seq") | (path | join("_")) + "='" + . + "'"), ( select(kind == "seq") | (path | join("_")) + "=(" + (map("'" + . + "'") | join(",")) + ")") )` var deepPruneExpression = `( .. | # recurse through all the nodes select(has("child1") or has("child2")) | # match parents that have either child1 or child2 (.child1, .child2) | # select those children select(.) # filter out nulls ) as $i ireduce({}; # using that set of nodes, create a new result map setpath($i | path; $i) # and put in each node, using its original path )` var recipes = []expressionScenario{ { description: "Find items in an array", subdescription: "We have an array and we want to find the elements with a particular name.", explanation: []string{ "`.[]` splats the array, and puts all the items in the context.", "These items are then piped (`|`) into `select(.name == \"Foo\")` which will select all the nodes that have a name property set to 'Foo'.", "See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information.", }, document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, expression: `.[] | select(.name == "Foo")`, expected: []string{ "D0, P[0], (!!map)::{name: Foo, numBuckets: 0}\n", }, }, { description: "Find and update items in an array", subdescription: "We have an array and we want to _update_ the elements with a particular name.", document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, expression: `(.[] | select(.name == "Foo") | .numBuckets) |= . + 1`, explanation: []string{ "Following from the example above`.[]` splats the array, selects filters the items.", "We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items", "Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression.", "`|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`).", "The expression `. + 1` increments the numBuckets counter.", "See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.", }, expected: []string{ "D0, P[], (!!seq)::[{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]\n", }, }, { description: "Deeply prune a tree", subdescription: "Say we are only interested in child1 and child2, and want to filter everything else out.", document: `{parentA: [bob],parentB: {child1: i am child1, child3: hiya},parentC: {childX: "cool",child2: me child2}}`, expression: deepPruneExpression, explanation: []string{ "Find all the matching child1 and child2 nodes", "Using ireduce, create a new map using just those nodes", "Set each node into the new map using its original path", }, expected: []string{ "D0, P[], (!!map)::parentB:\n child1: i am child1\nparentC:\n child2: me child2\n", }, }, { description: "Multiple or complex updates to items in an array", subdescription: "We have an array and we want to _update_ the elements with a particular name in reference to its type.", document: `myArray: [{name: Foo, type: cat}, {name: Bar, type: dog}]`, expression: `with(.myArray[]; .name = .name + " - " + .type)`, explanation: []string{ "The with operator will effectively loop through each given item in the first given expression, and run the second expression against it.", "`.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array", "`.name = .name + \" - \" + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type.", "See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples.", }, expected: []string{ "D0, P[], (!!map)::myArray: [{name: Foo - cat, type: cat}, {name: Bar - dog, type: dog}]\n", }, }, { description: "Sort an array by a field", document: `myArray: [{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]`, expression: `.myArray |= sort_by(.numBuckets)`, explanation: []string{ "We want to resort `.myArray`.", "`sort_by` works by piping an array into it, and it pipes out a sorted array.", "So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`", }, expected: []string{ "D0, P[], (!!map)::myArray: [{name: Bar, numBuckets: 0}, {name: Foo, numBuckets: 1}]\n", }, }, { description: "Filter, flatten, sort and unique", subdescription: "Lets find the unique set of names from the document.", document: `[{type: foo, names: [Fred, Catherine]}, {type: bar, names: [Zelda]}, {type: foo, names: Fred}, {type: foo, names: Ava}]`, expression: `[.[] | select(.type == "foo") | .names] | flatten | sort | unique`, explanation: []string{ "`.[] | select(.type == \"foo\") | .names` will select the array elements of type \"foo\"", "Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator.", "But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == \"foo\") | .names]`", "Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own.", "Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings", "Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!", "See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.", }, expected: []string{ "D0, P[], (!!seq)::- Ava\n- Catherine\n- Fred\n", }, }, { description: "Export as environment variables (script), or any custom format", subdescription: "Given a yaml document, lets output a script that will configure environment variables with that data. This same approach can be used for exporting into custom formats.", document: "var0: string0\nvar1: string1\nfruit: [apple, banana, peach]\n", expression: bashEnvScript, expected: []string{ "D0, P[var0='string0'], (!!str)::var0='string0'\n", "D0, P[var1='string1'], (!!str)::var1='string1'\n", "D0, P[fruit=('apple','banana','peach')], (!!str)::fruit=('apple','banana','peach')\n", }, explanation: []string{ "`.[]` matches all top level elements", "We need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together", "Scalars, we just need the key and quoted value: `( select(kind == \"scalar\") | key + \"='\" + . + \"'\")`", "Sequences (or arrays) are trickier, we need to quote each value and `join` them with `,`: `map(\"'\" + . + \"'\") | join(\",\")`", }, }, { description: "Custom format with nested data", subdescription: "Like the previous example, but lets handle nested data structures. In this custom example, we're going to join the property paths with _. The important thing to keep in mind is that our expression is not recursive (despite the data structure being so). Instead we match _all_ elements on the tree and operate on them.", document: "simple: string0\nsimpleArray: [apple, banana, peach]\ndeep:\n property: value\n array: [cat]\n", expression: nestedBashEnvScript, expected: []string{ "D0, P[simple], (!!str)::simple='string0'\n", "D0, P[deep property], (!!str)::deep_property='value'\n", "D0, P[simpleArray], (!!str)::simpleArray=('apple','banana','peach')\n", "D0, P[deep array], (!!str)::deep_array=('cat')\n", }, explanation: []string{ "You'll need to understand how the previous example works to understand this extension.", "`..` matches _all_ elements, instead of `.[]` from the previous example that just matches top level elements.", "Like before, we need a string expression for each of the different types that will produce the bash syntax, we'll use the union operator, to join them together", "This time, however, our expression matches every node in the data structure.", "We only want to print scalars that are not in arrays (because we handle the separately), so well add `and parent | kind != \"seq\"` to the select operator expression for scalars", "We don't just want the key any more, we want the full path. So instead of `key` we have `path | join(\"_\")`", "The expression for sequences follows the same logic", }, }, } func TestRecipes(t *testing.T) { for _, tt := range recipes { testScenario(t, &tt) } genericScenarios := make([]interface{}, len(recipes)) for i, s := range recipes { genericScenarios[i] = s } documentScenarios(t, "usage", "recipes", genericScenarios, documentOperatorScenario) } ================================================ FILE: pkg/yqlib/security_prefs.go ================================================ package yqlib type SecurityPreferences struct { DisableEnvOps bool DisableFileOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ DisableEnvOps: false, DisableFileOps: false, } ================================================ FILE: pkg/yqlib/shellvariables.go ================================================ package yqlib type ShellVariablesPreferences struct { KeySeparator string UnwrapScalar bool } func NewDefaultShellVariablesPreferences() ShellVariablesPreferences { return ShellVariablesPreferences{ KeySeparator: "_", UnwrapScalar: false, } } var ConfiguredShellVariablesPreferences = NewDefaultShellVariablesPreferences() ================================================ FILE: pkg/yqlib/shellvariables_test.go ================================================ package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) var shellVariablesScenarios = []formatScenario{ { description: "Encode shell variables", subdescription: "Note that comments are dropped and values will be enclosed in single quotes as needed.", input: "" + "# comment" + "\n" + "name: Mike Wazowski" + "\n" + "eyes:" + "\n" + " color: turquoise" + "\n" + " number: 1" + "\n" + "friends:" + "\n" + " - James P. Sullivan" + "\n" + " - Celia Mae", expected: "" + "name='Mike Wazowski'" + "\n" + "eyes_color=turquoise" + "\n" + "eyes_number=1" + "\n" + "friends_0='James P. Sullivan'" + "\n" + "friends_1='Celia Mae'" + "\n", }, { description: "Encode shell variables: illegal variable names as key.", subdescription: "Keys that would be illegal as variable keys are adapted.", input: "" + "ascii_=_symbols: replaced with _" + "\n" + "\"ascii_\t_controls\": dropped (this example uses \\t)" + "\n" + "nonascii_\u05d0_characters: dropped" + "\n" + "effort_expe\u00f1ded_t\u00f2_preserve_accented_latin_letters: moderate (via unicode NFKD)" + "\n", expected: "" + "ascii___symbols='replaced with _'" + "\n" + "ascii__controls='dropped (this example uses \\t)'" + "\n" + "nonascii__characters=dropped" + "\n" + "effort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'" + "\n", }, { description: "Encode shell variables: empty values, arrays and maps", subdescription: "Empty values are encoded to empty variables, but empty arrays and maps are skipped.", input: "empty:\n value:\n array: []\n map: {}", expected: "empty_value=" + "\n", }, { description: "Encode shell variables: single quotes in values", subdescription: "Single quotes in values are encoded as '\"'\"' (close single quote, double-quoted single quote, open single quote).", input: "name: Miles O'Brien", expected: `name='Miles O'"'"'Brien'` + "\n", }, { description: "Encode shell variables: custom separator", subdescription: "Use --shell-key-separator to specify a custom separator between keys. This is useful when the original keys contain underscores.", input: "" + "my_app:" + "\n" + " db_config:" + "\n" + " host: localhost" + "\n" + " port: 5432", expected: "" + "my_app__db_config__host=localhost" + "\n" + "my_app__db_config__port=5432" + "\n", scenarioType: "shell-separator", }, } func TestShellVariableScenarios(t *testing.T) { for _, s := range shellVariablesScenarios { //fmt.Printf("\t<%s> <%s>\n", s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder())) if s.scenarioType == "shell-separator" { // Save and restore the original separator originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator ConfiguredShellVariablesPreferences.KeySeparator = "__" test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description) ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator } else { test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()), s.description) } } genericScenarios := make([]interface{}, len(shellVariablesScenarios)) for i, s := range shellVariablesScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "shellvariables", genericScenarios, documentShellVariableScenario) } func documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if s.scenarioType == "shell-separator" { writeOrPanic(w, "```bash\nyq -o=shell --shell-key-separator=\"__\" sample.yml\n```\n") } else if expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=shell '%v' sample.yml\n```\n", expression)) } else { writeOrPanic(w, "```bash\nyq -o=shell sample.yml\n```\n") } writeOrPanic(w, "will output\n") if s.scenarioType == "shell-separator" { // Save and restore the original separator originalSeparator := ConfiguredShellVariablesPreferences.KeySeparator ConfiguredShellVariablesPreferences.KeySeparator = "__" writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))) ConfiguredShellVariablesPreferences.KeySeparator = originalSeparator } else { writeOrPanic(w, fmt.Sprintf("```sh\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewShellVariablesEncoder()))) } } ================================================ FILE: pkg/yqlib/stream_evaluator.go ================================================ package yqlib import ( "container/list" "errors" "fmt" "io" "os" ) // A yaml expression evaluator that runs the expression multiple times for each given yaml document. // Uses less memory than loading all documents and running the expression once, but this cannot process // cross document expressions. type StreamEvaluator interface { Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error EvaluateNew(expression string, printer Printer) error } type streamEvaluator struct { treeNavigator DataTreeNavigator fileIndex int } func NewStreamEvaluator() StreamEvaluator { return &streamEvaluator{treeNavigator: NewDataTreeNavigator()} } func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error { node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } candidateNode := createScalarNode(nil, "") inputList := list.New() inputList.PushBack(candidateNode) result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node) if errorParsing != nil { return errorParsing } return printer.PrintResults(result.MatchingNodes) } func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error { var totalProcessDocs uint node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } for _, filename := range filenames { reader, err := readStream(filename) if err != nil { return err } processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder) if err != nil { return err } totalProcessDocs = totalProcessDocs + processedDocs switch reader := reader.(type) { case *os.File: safelyCloseFile(reader) } } if totalProcessDocs == 0 { // problem is I've already slurped the leading content sadface return s.EvaluateNew(expression, printer) } return nil } func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) { var currentIndex uint err := decoder.Init(reader) if err != nil { return 0, err } for { candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { s.fileIndex = s.fileIndex + 1 return currentIndex, nil } else if errorReading != nil { return currentIndex, fmt.Errorf("bad file '%v': %w", filename, errorReading) } candidateNode.document = currentIndex candidateNode.filename = filename candidateNode.fileIndex = s.fileIndex inputList := list.New() inputList.PushBack(candidateNode) result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node) if errorParsing != nil { return currentIndex, errorParsing } err := printer.PrintResults(result.MatchingNodes) if err != nil { return currentIndex, err } currentIndex = currentIndex + 1 } } ================================================ FILE: pkg/yqlib/string_evaluator.go ================================================ package yqlib import ( "bufio" "bytes" "container/list" "strings" ) type StringEvaluator interface { Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) EvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error) } type stringEvaluator struct { treeNavigator DataTreeNavigator } func NewStringEvaluator() StringEvaluator { return &stringEvaluator{ treeNavigator: NewDataTreeNavigator(), } } func (s *stringEvaluator) EvaluateAll(expression string, input string, encoder Encoder, decoder Decoder) (string, error) { reader := bufio.NewReader(strings.NewReader(input)) var documents *list.List var results *list.List var err error if documents, err = ReadDocuments(reader, decoder); err != nil { return "", err } evaluator := NewAllAtOnceEvaluator() if results, err = evaluator.EvaluateCandidateNodes(expression, documents); err != nil { return "", err } out := new(bytes.Buffer) printer := NewPrinter(encoder, NewSinglePrinterWriter(out)) if err := printer.PrintResults(results); err != nil { return "", err } return out.String(), nil } func (s *stringEvaluator) Evaluate(expression string, input string, encoder Encoder, decoder Decoder) (string, error) { // Use bytes.Buffer for output of string out := new(bytes.Buffer) printer := NewPrinter(encoder, NewSinglePrinterWriter(out)) InitExpressionParser() node, err := ExpressionParser.ParseExpression(expression) if err != nil { return "", err } reader := bufio.NewReader(strings.NewReader(input)) evaluator := NewStreamEvaluator() if _, err := evaluator.Evaluate("", reader, node, printer, decoder); err != nil { return "", err } return out.String(), nil } ================================================ FILE: pkg/yqlib/string_evaluator_test.go ================================================ package yqlib import ( "testing" "github.com/mikefarah/yq/v4/test" ) func TestStringEvaluator_MultipleDocumentMerge(t *testing.T) { yamlString := "a: Hello\n---\na: Goodbye\n" expected_output := "a: Goodbye\n" encoder := NewYamlEncoder(ConfiguredYamlPreferences) decoder := NewYamlDecoder(ConfiguredYamlPreferences) result, err := NewStringEvaluator().EvaluateAll("select(di==0) * select(di==1)", yamlString, encoder, decoder) if err != nil { t.Error(err) } else { test.AssertResult(t, expected_output, result) } } func TestStringEvaluator_Evaluate_Nominal(t *testing.T) { expected_output := `` + `yq` + "\n" + `---` + "\n" + `jq` + "\n" expression := ".[].name" input := `` + ` - name: yq` + "\n" + ` description: yq is a portable command-line YAML, JSON and XML processor` + "\n" + `---` + "\n" + ` - name: jq` + "\n" + ` description: Command-line JSON processor` + "\n" encoder := NewYamlEncoder(ConfiguredYamlPreferences) decoder := NewYamlDecoder(ConfiguredYamlPreferences) result, err := NewStringEvaluator().Evaluate(expression, input, encoder, decoder) if err != nil { t.Error(err) } else { test.AssertResult(t, expected_output, result) } } ================================================ FILE: pkg/yqlib/toml.go ================================================ package yqlib type TomlPreferences struct { ColorsEnabled bool } func NewDefaultTomlPreferences() TomlPreferences { return TomlPreferences{ColorsEnabled: false} } func (p *TomlPreferences) Copy() TomlPreferences { return TomlPreferences{ColorsEnabled: p.ColorsEnabled} } var ConfiguredTomlPreferences = NewDefaultTomlPreferences() ================================================ FILE: pkg/yqlib/toml_test.go ================================================ package yqlib import ( "bufio" "bytes" "fmt" "strings" "testing" "github.com/fatih/color" "github.com/mikefarah/yq/v4/test" ) var sampleTable = ` var = "x" [owner.contact] name = "Tom Preston-Werner" age = 36 ` var tableArrayBeforeOwners = ` [[owner.addresses]] street = "first street" [owner] name = "Tom Preston-Werner" ` var expectedTableArrayBeforeOwners = `owner: addresses: - street: first street name: Tom Preston-Werner ` var sampleTableExpected = `var: x owner: contact: name: Tom Preston-Werner age: 36 ` var doubleArrayTable = ` [[fruits]] name = "apple" [[fruits.varieties]] # nested array of tables name = "red delicious"` var doubleArrayTableExpected = `fruits: - name: apple varieties: - name: red delicious ` var doubleArrayTableMultipleEntries = ` [[fruits]] name = "banana" [[fruits]] name = "apple" [[fruits.varieties]] # nested array of tables name = "red delicious"` var doubleArrayTableMultipleEntriesExpected = `fruits: - name: banana - name: apple varieties: - name: red delicious ` var doubleArrayTableNothingAbove = ` [[fruits.varieties]] # nested array of tables name = "red delicious"` var doubleArrayTableNothingAboveExpected = `fruits: varieties: - name: red delicious ` var doubleArrayTableEmptyAbove = ` [[fruits]] [[fruits.varieties]] # nested array of tables name = "red delicious"` var doubleArrayTableEmptyAboveExpected = `fruits: - varieties: - name: red delicious ` var emptyArrayTableThenTable = ` [[fruits]] [animals] [[fruits.varieties]] # nested array of tables name = "red delicious"` var emptyArrayTableThenTableExpected = `fruits: - varieties: - name: red delicious animals: {} ` var arrayTableThenArray = ` [[rootA.kidB]] cat = "meow" [rootA.kidB.kidC] dog = "bark"` var arrayTableThenArrayExpected = `rootA: kidB: - cat: meow kidC: dog: bark ` var sampleArrayTable = ` [owner.contact] name = "Tom Preston-Werner" age = 36 [[owner.addresses]] street = "first street" suburb = "ok" [[owner.addresses]] street = "second street" suburb = "nice" ` var sampleArrayTableExpected = `owner: contact: name: Tom Preston-Werner age: 36 addresses: - street: first street suburb: ok - street: second street suburb: nice ` var emptyTable = ` [dependencies] ` var emptyTableExpected = "dependencies: {}\n" var multipleEmptyTables = ` [firstEmptyTable] [firstTableWithContent] key = "value" [secondEmptyTable] [thirdEmptyTable] [secondTableWithContent] key = "value" [fourthEmptyTable] [fifthEmptyTable] ` var expectedMultipleEmptyTables = `firstEmptyTable: {} firstTableWithContent: key: value secondEmptyTable: {} thirdEmptyTable: {} secondTableWithContent: key: value fourthEmptyTable: {} fifthEmptyTable: {} ` var sampleWithHeader = ` [servers] [servers.alpha] ip = "10.0.0.1" ` var expectedSampleWithHeader = `servers: alpha: ip: 10.0.0.1 ` // Roundtrip fixtures var rtInlineTableAttr = `name = { first = "Tom", last = "Preston-Werner" } ` var rtTableSection = `[owner.contact] name = "Tom" age = 36 ` var rtArrayOfTables = `[[fruits]] name = "apple" [[fruits.varieties]] name = "red delicious" ` var rtArraysAndScalars = `A = ["hello", ["world", "again"]] B = 12 ` var rtSimple = `A = "hello" B = 12 ` var rtDeepPaths = `[person] name = "hello" address = "12 cat st" ` var rtEmptyArray = `A = [] ` var rtSampleTable = `var = "x" [owner.contact] name = "Tom Preston-Werner" age = 36 ` var rtEmptyTable = `[dependencies] ` var rtComments = `# This is a comment A = "hello" # inline comment B = 12 # Table comment [person] name = "Tom" # name comment ` // Reproduce bug for https://github.com/mikefarah/yq/issues/2588 // Bug: standalone comments inside a table cause subsequent key-values to be assigned at root. var issue2588RustToolchainWithComments = `[owner] # comment name = "Tomer" ` var tableWithComment = `[owner] # comment [things] ` var sampleFromWeb = `# This is a TOML document title = "TOML Example" [owner] name = "Tom Preston-Werner" dob = 1979-05-27T07:32:00-08:00 [database] enabled = true ports = [8000, 8001, 8002] data = [["delta", "phi"], [3.14]] temp_targets = { cpu = 79.5, case = 72.0 } # [servers] yq can't do this one yet [servers.alpha] ip = "10.0.0.1" role = "frontend" [servers.beta] ip = "10.0.0.2" role = "backend" ` var subArrays = ` [[array]] [[array.subarray]] [[array.subarray.subsubarray]] ` var tomlTableWithComments = `[section] the_array = [ # comment "value 1", # comment "value 2", ] ` var expectedSubArrays = `array: - subarray: - subsubarray: - {} ` var tomlScenarios = []formatScenario{ { skipDoc: true, description: "blank", input: "", expected: "", scenarioType: "decode", }, { skipDoc: true, description: "table array before owners", input: tableArrayBeforeOwners, expected: expectedTableArrayBeforeOwners, scenarioType: "decode", }, { skipDoc: true, description: "datetime", input: "A = 1979-05-27T07:32:00-08:00", expected: "A: 1979-05-27T07:32:00-08:00\n", scenarioType: "decode", }, { skipDoc: true, description: "blank", input: `A = "hello`, expectedError: `bad file 'sample.yml': basic string not terminated by "`, scenarioType: "decode-error", }, { description: "Parse: Simple", input: "A = \"hello\"\nB = 12\n", expected: "A: hello\nB: 12\n", scenarioType: "decode", }, { description: "Parse: Deep paths", input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n", expected: "person:\n name: hello\n address: 12 cat st\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse: include key information", input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n", expression: ".person.name | key", expected: "name\n", scenarioType: "roundtrip", }, { skipDoc: true, description: "Parse: include parent information", input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n", expression: ".person.name | parent", expected: "name: hello\naddress: 12 cat st\n", scenarioType: "decode", }, { skipDoc: true, description: "Parse: include path information", input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n", expression: ".person.name | path", expected: "- person\n- name\n", scenarioType: "decode", }, { description: "Encode: Scalar", input: "person.name = \"hello\"\nperson.address = \"12 cat st\"\n", expression: ".person.name", expected: "hello\n", scenarioType: "roundtrip", }, { skipDoc: true, input: `A.B = "hello"`, expected: "A:\n B: hello\n", scenarioType: "decode", }, { skipDoc: true, description: "bool", input: `A = true`, expected: "A: true\n", scenarioType: "decode", }, { skipDoc: true, description: "bool false", input: `A = false `, expected: "A: false\n", scenarioType: "decode", }, { skipDoc: true, description: "number", input: `A = 3 `, expected: "A: 3\n", scenarioType: "decode", }, { skipDoc: true, description: "number", input: `A = 0xDEADBEEF`, expression: " .A += 1", expected: "A: 0xDEADBEF0\n", scenarioType: "decode", }, { skipDoc: true, description: "float", input: `A = 6.626e-34`, expected: "A: 6.626e-34\n", scenarioType: "decode", }, { skipDoc: true, description: "empty arraY", input: `A = []`, expected: "A: []\n", scenarioType: "decode", }, { skipDoc: true, description: "array", input: `A = ["hello", ["world", "again"]]`, expected: "A:\n - hello\n - - world\n - again\n", scenarioType: "decode", }, { description: "Parse: inline table", input: `name = { first = "Tom", last = "Preston-Werner" }`, expected: "name:\n first: Tom\n last: Preston-Werner\n", scenarioType: "decode", }, { skipDoc: true, input: sampleTable, expected: sampleTableExpected, scenarioType: "decode", }, { description: "Parse: Array Table", input: sampleArrayTable, expected: sampleArrayTableExpected, scenarioType: "decode", }, { description: "Parse: Array of Array Table", input: doubleArrayTable, expected: doubleArrayTableExpected, scenarioType: "decode", }, { skipDoc: true, description: "Parse: Array of Array Table; nothing above", input: doubleArrayTableNothingAbove, expected: doubleArrayTableNothingAboveExpected, scenarioType: "decode", }, { skipDoc: true, description: "Parse: Array of Array Table; empty above", input: doubleArrayTableEmptyAbove, expected: doubleArrayTableEmptyAboveExpected, scenarioType: "decode", }, { skipDoc: true, description: "Parse: Array of Array Table; multiple entries", input: doubleArrayTableMultipleEntries, expected: doubleArrayTableMultipleEntriesExpected, scenarioType: "decode", }, { skipDoc: true, description: "Parse: Array of Array Table; then table; then array table", input: emptyArrayTableThenTable, expected: emptyArrayTableThenTableExpected, scenarioType: "decode", }, { skipDoc: true, description: "Parse: Array of Array Table; then table", input: arrayTableThenArray, expected: arrayTableThenArrayExpected, scenarioType: "decode", }, { description: "Parse: Empty Table", input: emptyTable, expected: emptyTableExpected, scenarioType: "decode", }, { description: "Parse: with header", skipDoc: true, input: sampleWithHeader, expected: expectedSampleWithHeader, scenarioType: "decode", }, { description: "Parse: multiple empty tables", skipDoc: true, input: multipleEmptyTables, expected: expectedMultipleEmptyTables, scenarioType: "decode", }, { description: "subArrays", skipDoc: true, input: subArrays, expected: expectedSubArrays, scenarioType: "decode", }, // Roundtrip scenarios { description: "Roundtrip: inline table attribute", input: rtInlineTableAttr, expression: ".", expected: rtInlineTableAttr, scenarioType: "roundtrip", }, { description: "Roundtrip: table section", input: rtTableSection, expression: ".", expected: rtTableSection, scenarioType: "roundtrip", }, { description: "Roundtrip: array of tables", input: rtArrayOfTables, expression: ".", expected: rtArrayOfTables, scenarioType: "roundtrip", }, { description: "Roundtrip: arrays and scalars", input: rtArraysAndScalars, expression: ".", expected: rtArraysAndScalars, scenarioType: "roundtrip", }, { description: "Roundtrip: simple", input: rtSimple, expression: ".", expected: rtSimple, scenarioType: "roundtrip", }, { description: "Roundtrip: deep paths", input: rtDeepPaths, expression: ".", expected: rtDeepPaths, scenarioType: "roundtrip", }, { description: "Roundtrip: empty array", input: rtEmptyArray, expression: ".", expected: rtEmptyArray, scenarioType: "roundtrip", }, { description: "Roundtrip: sample table", input: rtSampleTable, expression: ".", expected: rtSampleTable, scenarioType: "roundtrip", }, { description: "Roundtrip: empty table", input: rtEmptyTable, expression: ".", expected: rtEmptyTable, scenarioType: "roundtrip", }, { description: "Roundtrip: comments", input: rtComments, expression: ".", expected: rtComments, scenarioType: "roundtrip", }, { skipDoc: true, description: "Issue #2588: comments inside table must not flatten (.owner.name)", input: issue2588RustToolchainWithComments, expression: ".owner.name", expected: "Tomer\n", scenarioType: "decode", }, { skipDoc: true, description: "Issue #2588: comments inside table must not flatten (.name)", input: issue2588RustToolchainWithComments, expression: ".name", expected: "null\n", scenarioType: "decode", }, { skipDoc: true, input: issue2588RustToolchainWithComments, expected: issue2588RustToolchainWithComments, scenarioType: "roundtrip", }, { skipDoc: true, input: tableWithComment, expression: ".owner | headComment", expected: "comment\n", scenarioType: "roundtrip", }, { description: "Roundtrip: sample from web", input: sampleFromWeb, expression: ".", expected: sampleFromWeb, scenarioType: "roundtrip", }, { skipDoc: true, input: tomlTableWithComments, expected: tomlTableWithComments, scenarioType: "roundtrip", }, } func testTomlScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "decode-error": result, err := processFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description) } } func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.toml file of:\n") writeOrPanic(w, fmt.Sprintf("```toml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.toml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.toml file of:\n") writeOrPanic(w, fmt.Sprintf("```toml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if expression == "" { expression = "." } writeOrPanic(w, fmt.Sprintf("```bash\nyq '%v' sample.toml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()))) } func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "decode": documentTomlDecodeScenario(w, s) case "roundtrip": documentTomlRoundtripScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func TestTomlScenarios(t *testing.T) { for _, tt := range tomlScenarios { testTomlScenario(t, tt) } genericScenarios := make([]interface{}, len(tomlScenarios)) for i, s := range tomlScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario) } // TestTomlColourization tests that colourization correctly distinguishes // between table section headers and inline arrays func TestTomlColourization(t *testing.T) { // Save and restore color state oldNoColor := color.NoColor color.NoColor = false defer func() { color.NoColor = oldNoColor }() // Test that inline arrays are not coloured as table sections encoder := &tomlEncoder{prefs: TomlPreferences{ColorsEnabled: true}} // Create TOML with both table sections and inline arrays input := []byte(`[database] enabled = true ports = [8000, 8001, 8002] [servers] alpha = "test" `) result := encoder.colorizeToml(input) resultStr := string(result) // The bug would cause the inline array [8000, 8001, 8002] to be // coloured with the section colour (Yellow + Bold) instead of being // left uncoloured or coloured differently. // // To test this, we check that the section colour codes appear only // for actual table sections, not for inline arrays. // Get the ANSI codes for section colour (Yellow + Bold) sectionColourObj := color.New(color.FgYellow, color.Bold) sectionColourObj.EnableColor() sampleSection := sectionColourObj.Sprint("[database]") // Extract just the ANSI codes from the sample // ANSI codes start with \x1b[ var ansiStart string for i := 0; i < len(sampleSection); i++ { if sampleSection[i] == '\x1b' { // Find the end of the ANSI sequence (ends with 'm') end := i for end < len(sampleSection) && sampleSection[end] != 'm' { end++ } if end < len(sampleSection) { ansiStart = sampleSection[i : end+1] break } } } // Count how many times the section colour appears in the output // It should appear exactly twice: once for [database] and once for [servers] // If it appears more times (e.g., for [8000, 8001, 8002]), that's the bug sectionColourCount := strings.Count(resultStr, ansiStart) // We expect exactly 2 occurrences (for [database] and [servers]) // The bug would cause more occurrences (e.g., also for [8000) if sectionColourCount != 2 { t.Errorf("Expected section colour to appear exactly 2 times (for [database] and [servers]), but it appeared %d times.\nOutput: %s", sectionColourCount, resultStr) } } func TestTomlColorisationNumberBug(t *testing.T) { // Save and restore color state oldNoColor := color.NoColor color.NoColor = false defer func() { color.NoColor = oldNoColor }() encoder := NewTomlEncoder() tomlEncoder := encoder.(*tomlEncoder) // Test case that exposes the bug: "123-+-45" should NOT be colourised as a single number input := "A = 123-+-45\n" result := string(tomlEncoder.colorizeToml([]byte(input))) // The bug causes "123-+-45" to be colourised as one token // It should stop at "123" because the next character '-' is not valid in this position if strings.Contains(result, "123-+-45") { // Check if it's colourised as a single token (no color codes in the middle) idx := strings.Index(result, "123-+-45") // Look backwards for color code beforeIdx := idx - 1 for beforeIdx >= 0 && result[beforeIdx] != '\x1b' { beforeIdx-- } // Look forward for reset code afterIdx := idx + 8 // length of "123-+-45" hasResetAfter := false for afterIdx < len(result) && afterIdx < idx+20 { if result[afterIdx] == '\x1b' { hasResetAfter = true break } afterIdx++ } if beforeIdx >= 0 && hasResetAfter { // The entire "123-+-45" is wrapped in color codes - this is the bug! t.Errorf("BUG DETECTED: '123-+-45' is incorrectly colourised as a single number") t.Errorf("Expected only '123' to be colourised as a number, but got the entire '123-+-45'") t.Logf("Full output: %q", result) t.Fail() } } // Additional test cases for the bug bugTests := []struct { name string input string invalidSequence string description string }{ { name: "consecutive minuses", input: "A = 123--45\n", invalidSequence: "123--45", description: "'123--45' should not be colourised as a single number", }, { name: "plus in middle", input: "A = 123+45\n", invalidSequence: "123+45", description: "'123+45' should not be colourised as a single number", }, } for _, tt := range bugTests { t.Run(tt.name, func(t *testing.T) { result := string(tomlEncoder.colorizeToml([]byte(tt.input))) if strings.Contains(result, tt.invalidSequence) { idx := strings.Index(result, tt.invalidSequence) beforeIdx := idx - 1 for beforeIdx >= 0 && result[beforeIdx] != '\x1b' { beforeIdx-- } afterIdx := idx + len(tt.invalidSequence) hasResetAfter := false for afterIdx < len(result) && afterIdx < idx+20 { if result[afterIdx] == '\x1b' { hasResetAfter = true break } afterIdx++ } if beforeIdx >= 0 && hasResetAfter { t.Errorf("BUG: %s", tt.description) t.Logf("Full output: %q", result) } } }) } // Test that valid scientific notation still works validTests := []struct { name string input string }{ {"scientific positive", "A = 1.23e+45\n"}, {"scientific negative", "A = 6.626e-34\n"}, {"scientific uppercase", "A = 1.23E+10\n"}, } for _, tt := range validTests { t.Run(tt.name, func(t *testing.T) { result := tomlEncoder.colorizeToml([]byte(tt.input)) if len(result) == 0 { t.Error("Expected non-empty colourised output") } }) } } // Tests that the encoder handles empty path slices gracefully func TestTomlEmptyPathPanic(t *testing.T) { encoder := NewTomlEncoder() tomlEncoder := encoder.(*tomlEncoder) var buf bytes.Buffer // Create a simple scalar node scalarNode := &CandidateNode{ Kind: ScalarNode, Tag: "!!str", Value: "test", } // Test with empty path - this should not panic err := tomlEncoder.encodeTopLevelEntry(&buf, []string{}, scalarNode) if err == nil { t.Error("Expected error when encoding with empty path, got nil") } } // TestTomlStringEscapeColourization tests that string colourization correctly // handles escape sequences, particularly escaped quotes at the end of strings func TestTomlStringEscapeColourization(t *testing.T) { // Save and restore color state oldNoColor := color.NoColor color.NoColor = false defer func() { color.NoColor = oldNoColor }() encoder := NewTomlEncoder() tomlEncoder := encoder.(*tomlEncoder) testCases := []struct { name string input string description string }{ { name: "escaped quote at end", input: `A = "test\""` + "\n", description: "String ending with escaped quote should be colourised correctly", }, { name: "escaped backslash then quote", input: `A = "test\\\""` + "\n", description: "String with escaped backslash followed by escaped quote", }, { name: "escaped quote in middle", input: `A = "test\"middle"` + "\n", description: "String with escaped quote in the middle should be colourised correctly", }, { name: "multiple escaped quotes", input: `A = "\"test\""` + "\n", description: "String with escaped quotes at start and end", }, { name: "escaped newline", input: `A = "test\n"` + "\n", description: "String with escaped newline should be colourised correctly", }, { name: "single quote with escaped single quote", input: `A = 'test\''` + "\n", description: "Single-quoted string with escaped single quote", }, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { // The test should not panic and should return some output result := tomlEncoder.colorizeToml([]byte(tt.input)) if len(result) == 0 { t.Error("Expected non-empty colourised output") } // Check that the result contains the input string (with color codes) // At minimum, it should contain "A" and "=" resultStr := string(result) if !strings.Contains(resultStr, "A") || !strings.Contains(resultStr, "=") { t.Errorf("Expected output to contain 'A' and '=', got: %q", resultStr) } }) } } func TestTomlEncoderPrintDocumentSeparator(t *testing.T) { encoder := NewTomlEncoder() var buf bytes.Buffer writer := bufio.NewWriter(&buf) err := encoder.PrintDocumentSeparator(writer) writer.Flush() test.AssertResult(t, nil, err) test.AssertResult(t, "", buf.String()) } func TestTomlEncoderPrintLeadingContent(t *testing.T) { encoder := NewTomlEncoder() var buf bytes.Buffer writer := bufio.NewWriter(&buf) err := encoder.PrintLeadingContent(writer, "some content") writer.Flush() test.AssertResult(t, nil, err) test.AssertResult(t, "", buf.String()) } func TestTomlEncoderCanHandleAliases(t *testing.T) { encoder := NewTomlEncoder() test.AssertResult(t, false, encoder.CanHandleAliases()) } ================================================ FILE: pkg/yqlib/utils.go ================================================ package yqlib import ( "bufio" "container/list" "errors" "fmt" "io" "os" ) func readStream(filename string) (io.Reader, error) { var reader *bufio.Reader if filename == "-" { reader = bufio.NewReader(os.Stdin) } else { // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside that directory. file, err := os.Open(filename) // #nosec if err != nil { return nil, err } reader = bufio.NewReader(file) } return reader, nil } func writeString(writer io.Writer, txt string) error { _, errorWriting := writer.Write([]byte(txt)) return errorWriting } func ReadDocuments(reader io.Reader, decoder Decoder) (*list.List, error) { return readDocuments(reader, "", 0, decoder) } func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) { err := decoder.Init(reader) if err != nil { return nil, err } inputList := list.New() var currentIndex uint for { candidateNode, errorReading := decoder.Decode() if errors.Is(errorReading, io.EOF) { switch reader := reader.(type) { case *os.File: safelyCloseFile(reader) } return inputList, nil } else if errorReading != nil { return nil, fmt.Errorf("bad file '%v': %w", filename, errorReading) } candidateNode.document = currentIndex candidateNode.filename = filename candidateNode.fileIndex = fileIndex candidateNode.EvaluateTogether = true inputList.PushBack(candidateNode) currentIndex = currentIndex + 1 } } ================================================ FILE: pkg/yqlib/write_in_place_handler.go ================================================ package yqlib import ( "os" ) type writeInPlaceHandler interface { CreateTempFile() (*os.File, error) FinishWriteInPlace(evaluatedSuccessfully bool) error } type writeInPlaceHandlerImpl struct { inputFilename string tempFile *os.File } func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler { return &writeInPlaceHandlerImpl{inputFile, nil} } func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) { file, err := createTempFile() if err != nil { return nil, err } info, err := os.Stat(w.inputFilename) if err != nil { return nil, err } err = os.Chmod(file.Name(), info.Mode()) if err != nil { return nil, err } if err = changeOwner(info, file); err != nil { return nil, err } log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name()) w.tempFile = file return file, err } func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error { log.Debug("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) safelyCloseFile(w.tempFile) if evaluatedSuccessfully { log.Debug("Moving temp file to target") return tryRenameFile(w.tempFile.Name(), w.inputFilename) } tryRemoveTempFile(w.tempFile.Name()) return nil } ================================================ FILE: pkg/yqlib/write_in_place_handler_test.go ================================================ package yqlib import ( "os" "path/filepath" "testing" ) func TestWriteInPlaceHandlerImpl_CreateTempFile(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "input.yaml") // Create input file with some content content := []byte("test: value\n") err := os.WriteFile(inputFile, content, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } handler := NewWriteInPlaceHandler(inputFile) tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } if tempFile == nil { t.Fatal("CreateTempFile returned nil file") } // Clean up tempFile.Close() os.Remove(tempFile.Name()) } func TestWriteInPlaceHandlerImpl_CreateTempFile_NonExistentInput(t *testing.T) { // Test with non-existent input file handler := NewWriteInPlaceHandler("/non/existent/file.yaml") tempFile, err := handler.CreateTempFile() if err == nil { t.Error("Expected error for non-existent input file, got nil") } if tempFile != nil { t.Error("Expected nil temp file for non-existent input file") tempFile.Close() } } func TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Success(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "input.yaml") // Create input file with some content content := []byte("test: value\n") err := os.WriteFile(inputFile, content, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } handler := NewWriteInPlaceHandler(inputFile) tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } defer tempFile.Close() // Write some content to temp file tempContent := []byte("updated: content\n") _, err = tempFile.Write(tempContent) if err != nil { t.Fatalf("Failed to write to temp file: %v", err) } tempFile.Close() // Test successful finish err = handler.FinishWriteInPlace(true) if err != nil { t.Fatalf("FinishWriteInPlace failed: %v", err) } // Verify the original file was updated updatedContent, err := os.ReadFile(inputFile) if err != nil { t.Fatalf("Failed to read updated file: %v", err) } if string(updatedContent) != string(tempContent) { t.Errorf("File content not updated correctly. Expected %q, got %q", string(tempContent), string(updatedContent)) } } func TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Failure(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "input.yaml") // Create input file with some content content := []byte("test: value\n") err := os.WriteFile(inputFile, content, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } handler := NewWriteInPlaceHandler(inputFile) tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } defer tempFile.Close() // Write some content to temp file tempContent := []byte("updated: content\n") _, err = tempFile.Write(tempContent) if err != nil { t.Fatalf("Failed to write to temp file: %v", err) } tempFile.Close() // Test failure finish (should not update the original file) err = handler.FinishWriteInPlace(false) if err != nil { t.Fatalf("FinishWriteInPlace failed: %v", err) } // Verify the original file was NOT updated originalContent, err := os.ReadFile(inputFile) if err != nil { t.Fatalf("Failed to read original file: %v", err) } if string(originalContent) != string(content) { t.Errorf("File content should not have been updated. Expected %q, got %q", string(content), string(originalContent)) } } func TestWriteInPlaceHandlerImpl_FinishWriteInPlace_Symlink_Success(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "input.yaml") symlinkFile := filepath.Join(tempDir, "symlink.yaml") // Create input file with some content content := []byte("test: value\n") err := os.WriteFile(inputFile, content, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } err = os.Symlink(inputFile, symlinkFile) if err != nil { t.Fatalf("Failed to symlink to input file: %v", err) } handler := NewWriteInPlaceHandler(symlinkFile) tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } defer tempFile.Close() // Write some content to temp file tempContent := []byte("updated: content\n") _, err = tempFile.Write(tempContent) if err != nil { t.Fatalf("Failed to write to temp file: %v", err) } tempFile.Close() // Test successful finish err = handler.FinishWriteInPlace(true) if err != nil { t.Fatalf("FinishWriteInPlace failed: %v", err) } // Verify that the symlink is still present info, err := os.Lstat(symlinkFile) if err != nil { t.Fatalf("Failed to lstat input file: %v", err) } if info.Mode()&os.ModeSymlink == 0 { t.Errorf("Input file symlink is no longer present") } // Verify the original file was updated updatedContent, err := os.ReadFile(inputFile) if err != nil { t.Fatalf("Failed to read updated file: %v", err) } if string(updatedContent) != string(tempContent) { t.Errorf("File content not updated correctly. Expected %q, got %q", string(tempContent), string(updatedContent)) } } func TestWriteInPlaceHandlerImpl_CreateTempFile_Permissions(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "input.yaml") // Create input file with specific permissions content := []byte("test: value\n") err := os.WriteFile(inputFile, content, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } handler := NewWriteInPlaceHandler(inputFile) tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } defer tempFile.Close() // Check that temp file has same permissions as input file tempFileInfo, err := os.Stat(tempFile.Name()) if err != nil { t.Fatalf("Failed to stat temp file: %v", err) } inputFileInfo, err := os.Stat(inputFile) if err != nil { t.Fatalf("Failed to stat input file: %v", err) } if tempFileInfo.Mode() != inputFileInfo.Mode() { t.Errorf("Temp file permissions don't match input file. Expected %v, got %v", inputFileInfo.Mode(), tempFileInfo.Mode()) } } func TestWriteInPlaceHandlerImpl_Integration(t *testing.T) { // Create a temporary directory and file for testing tempDir := t.TempDir() inputFile := filepath.Join(tempDir, "integration_test.yaml") // Create input file with some content originalContent := []byte("original: content\n") err := os.WriteFile(inputFile, originalContent, 0600) if err != nil { t.Fatalf("Failed to create input file: %v", err) } handler := NewWriteInPlaceHandler(inputFile) // Create temp file tempFile, err := handler.CreateTempFile() if err != nil { t.Fatalf("CreateTempFile failed: %v", err) } // Write new content to temp file newContent := []byte("new: content\n") _, err = tempFile.Write(newContent) if err != nil { t.Fatalf("Failed to write to temp file: %v", err) } tempFile.Close() // Finish with success err = handler.FinishWriteInPlace(true) if err != nil { t.Fatalf("FinishWriteInPlace failed: %v", err) } // Verify the file was updated finalContent, err := os.ReadFile(inputFile) if err != nil { t.Fatalf("Failed to read final file: %v", err) } if string(finalContent) != string(newContent) { t.Errorf("File not updated correctly. Expected %q, got %q", string(newContent), string(finalContent)) } } ================================================ FILE: pkg/yqlib/xml.go ================================================ package yqlib type XmlPreferences struct { Indent int AttributePrefix string ContentName string StrictMode bool KeepNamespace bool UseRawToken bool ProcInstPrefix string DirectiveName string SkipProcInst bool SkipDirectives bool } func NewDefaultXmlPreferences() XmlPreferences { return XmlPreferences{ Indent: 2, AttributePrefix: "+@", ContentName: "+content", StrictMode: false, KeepNamespace: true, UseRawToken: true, ProcInstPrefix: "+p_", DirectiveName: "+directive", SkipProcInst: false, SkipDirectives: false, } } func (p *XmlPreferences) Copy() XmlPreferences { return XmlPreferences{ Indent: p.Indent, AttributePrefix: p.AttributePrefix, ContentName: p.ContentName, StrictMode: p.StrictMode, KeepNamespace: p.KeepNamespace, UseRawToken: p.UseRawToken, ProcInstPrefix: p.ProcInstPrefix, DirectiveName: p.DirectiveName, SkipProcInst: p.SkipProcInst, SkipDirectives: p.SkipDirectives, } } var ConfiguredXMLPreferences = NewDefaultXmlPreferences() ================================================ FILE: pkg/yqlib/xml_test.go ================================================ //go:build !yq_noxml package yqlib import ( "bufio" "fmt" "testing" "github.com/mikefarah/yq/v4/test" ) const yamlInputWithProcInstAndHeadComment = `# cats +p_xml: version="1.0" this: is some xml` const expectedXmlProcInstAndHeadComment = ` is some xml ` const xmlProcInstAndHeadCommentBlock = ` is some xml ` const expectedYamlProcInstAndHeadCommentBlock = `# # cats # +p_xml: version="1.0" this: is some xml ` const inputXMLWithComments = ` 3 z ` const inputXMLWithCommentsWithSubChild = ` 3 ` const expectedDecodeYamlWithSubChild = `# before cat cat: # in cat before x: "3" # multi # line comment # for x # before y y: # in y before d: # in d before z: +@sweet: cool # in d after # in y after # in_cat_after # after cat ` const inputXMLWithCommentsWithArray = ` 3 ` const expectedDecodeYamlWithArray = `# before cat cat: # in cat before x: "3" # multi # line comment # for x # before y y: # in y before d: - # in d before z: +@sweet: cool # in d after - # in d2 before z: +@sweet: cool2 # in d2 after # in y after # in_cat_after # after cat ` const expectedDecodeYamlWithComments = `# before cat cat: # in cat before x: "3" # multi # line comment # for x # before y y: # in y before # in d before d: z # in d after # in y after # in_cat_after # after cat ` const expectedRoundtripXMLWithComments = ` 3 z ` const yamlWithComments = `# # header comment # above_cat # cat: # inline_cat # above_array array: # inline_array - val1 # inline_val1 # above_val2 - val2 # inline_val2 # below_cat ` const expectedXMLWithComments = ` val1 val2 ` const inputXMLWithNamespacedAttr = ` baz foobar ` const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0" map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@xsi:schemaLocation: some-url item: +content: baz +@foo: bar xsi:item: foobar ` const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0" map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@xsi:schemaLocation: some-url item: +content: baz +@foo: bar xsi:item: foobar ` const expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version="1.0" some-namespace:map: +@xmlns: some-namespace +@xmlns:xsi: some-instance +@some-instance:schemaLocation: some-url some-namespace:item: +content: baz +@foo: bar some-instance:item: foobar ` const xmlWithCustomDtd = ` ]> &writer;©right; ` const expectedDtd = ` ]> &writer;&copyright; ` const expectedSkippedDtd = ` &writer;&copyright; ` const xmlWithProcInstAndDirectives = ` things ` const yamlWithProcInstAndDirectives = `+p_xml: version="1.0" +directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" ' apple: +p_coolioo: version="1.0" +directive: 'CATYPE meow purr puss ' b: things ` const expectedXmlWithProcInstAndDirectives = ` things ` var xmlScenarios = []formatScenario{ { skipDoc: true, description: "bad xml", input: ``, expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\n", }, { skipDoc: true, input: " value ", expected: "root: value # comment\n", }, { skipDoc: true, input: "valuevalue", expectedError: "bad file 'sample.yml': invalid XML: Encountered chardata [value] outside of XML node", scenarioType: "decode-error", }, { skipDoc: true, input: "value", expected: "# comment\nroot: value\n", }, { skipDoc: true, input: " ", expected: "root: # comment\n", }, { skipDoc: true, input: "valueanotherValue ", expected: "root:\n # comment\n - value\n - anotherValue\n", }, { skipDoc: true, input: "quicksoftsquishy", expected: "root:\n cats:\n cat:\n - quick\n - soft\n # kitty_comment\n\n - squishy\n", }, { description: "ProcInst with head comment", skipDoc: true, input: yamlInputWithProcInstAndHeadComment, expected: expectedXmlProcInstAndHeadComment, scenarioType: "encode", }, { description: "Scalar roundtrip", skipDoc: true, input: "cat", expression: ".mike", expected: "cat", scenarioType: "roundtrip", }, { description: "ProcInst with head comment round trip", skipDoc: true, input: expectedXmlProcInstAndHeadComment, expected: expectedXmlProcInstAndHeadComment, scenarioType: "roundtrip", }, { description: "ProcInst with block head comment to yaml", skipDoc: true, input: xmlProcInstAndHeadCommentBlock, expected: expectedYamlProcInstAndHeadCommentBlock, scenarioType: "decode", }, { description: "ProcInst with block head comment from yaml", skipDoc: true, input: expectedYamlProcInstAndHeadCommentBlock, expected: xmlProcInstAndHeadCommentBlock, scenarioType: "encode", }, { description: "ProcInst with head comment round trip block", skipDoc: true, input: xmlProcInstAndHeadCommentBlock, expected: xmlProcInstAndHeadCommentBlock, scenarioType: "roundtrip", }, { description: "Parse xml: simple", subdescription: "Notice how all the values are strings, see the next example on how you can fix that.", input: "\n\n meow\n 4\n true\n", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: \"4\"\n cute: \"true\"\n", }, { description: "Parse xml: number", subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.", input: "\n\n meow\n 4\n true\n", expression: " (.. | select(tag == \"!!str\")) |= from_yaml", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n says: meow\n legs: 4\n cute: true\n", }, { description: "Parse xml: array", subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.", input: "\ncat\ngoat", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n", }, { description: "Parse xml: force as an array", subdescription: "In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.", input: "cat", expression: ".zoo.animal |= ([] + .)", expected: "zoo:\n animal:\n - cat\n", }, { description: "Parse xml: force all as an array", input: "boing", expression: ".. |= [] + .", expected: "- zoo:\n - thing:\n - frog:\n - boing\n", }, { description: "Parse xml: attributes", subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.", input: "\n\n 7\n", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +@legs: \"4\"\n legs: \"7\"\n", }, { description: "Parse xml: attributes with content", subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.", input: "\nmeow", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n +content: meow\n +@legs: \"4\"\n", }, { description: "Parse xml: content split between comments/children", subdescription: "Multiple content texts are collected into a sequence.", input: " value anotherValue frog cool!", expected: "root:\n +content: # comment\n - value\n - anotherValue\n - cool!\n a: frog\n", }, { description: "Parse xml: custom dtd", subdescription: "DTD entities are processed as directives.", input: xmlWithCustomDtd, expected: expectedDtd, scenarioType: "roundtrip", }, { description: "Roundtrip with name spaced attributes", skipDoc: true, input: inputXMLWithNamespacedAttr, expected: inputXMLWithNamespacedAttr, scenarioType: "roundtrip", }, { description: "Parse xml: skip custom dtd", subdescription: "DTDs are directives, skip over directives to skip DTDs.", input: xmlWithCustomDtd, expected: expectedSkippedDtd, scenarioType: "roundtrip-skip-directives", }, { description: "Parse xml: with comments", subdescription: "A best attempt is made to preserve comments.", input: inputXMLWithComments, expected: expectedDecodeYamlWithComments, scenarioType: "decode", }, { description: "Empty doc", skipDoc: true, input: "", expected: "\n", scenarioType: "decode", }, { description: "Empty single node", skipDoc: true, input: "", expected: "a:\n", scenarioType: "decode", }, { description: "Empty close node", skipDoc: true, input: "", expected: "a:\n", scenarioType: "decode", }, { description: "Nested empty", skipDoc: true, input: "", expected: "a:\n b:\n", scenarioType: "decode", }, { description: "Parse xml: with comments subchild", skipDoc: true, input: inputXMLWithCommentsWithSubChild, expected: expectedDecodeYamlWithSubChild, scenarioType: "decode", }, { description: "Parse xml: with comments array", skipDoc: true, input: inputXMLWithCommentsWithArray, expected: expectedDecodeYamlWithArray, scenarioType: "decode", }, { description: "Parse xml: keep attribute namespace", subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.KeepNamespace), skipDoc: false, input: inputXMLWithNamespacedAttr, expected: expectedYAMLWithNamespacedAttr, scenarioType: "decode-keep-ns", }, { description: "Parse xml: keep raw attribute namespace", skipDoc: true, input: inputXMLWithNamespacedAttr, expected: expectedYAMLWithRawNamespacedAttr, scenarioType: "decode-raw-token", }, { description: "Parse xml: keep raw attribute namespace", subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.UseRawToken), skipDoc: false, input: inputXMLWithNamespacedAttr, expected: expectedYAMLWithoutRawNamespacedAttr, scenarioType: "decode-raw-token-off", }, { description: "Encode xml: simple", input: "cat: purrs", expected: "purrs\n", scenarioType: "encode", }, { description: "includes map tags", skipDoc: true, input: "purrs\n", expression: `tag`, expected: "!!map\n", scenarioType: "decode", }, { description: "includes array tags", skipDoc: true, input: "purrspurrs\n", expression: `.cat | tag`, expected: "!!seq\n", scenarioType: "decode", }, { description: "Encode xml: array", input: "pets:\n cat:\n - purrs\n - meows", expected: "\n purrs\n meows\n\n", scenarioType: "encode", }, { description: "Encode xml: attributes", subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.", input: "cat:\n +@name: tiger\n meows: true\n", expected: "\n true\n\n", scenarioType: "encode", }, { description: "double prefix", skipDoc: true, input: "cat:\n +@+@name: tiger\n meows: true\n", expected: "\n true\n\n", scenarioType: "encode", }, { description: "arrays cannot be encoded", skipDoc: true, input: "[cat, dog, fish]", expectedError: "cannot encode !!seq to XML - only maps can be encoded", scenarioType: "encode-error", }, { description: "arrays cannot be encoded - 2", skipDoc: true, input: "[cat, dog]", expectedError: "cannot encode !!seq to XML - only maps can be encoded", scenarioType: "encode-error", }, { description: "Encode xml: attributes with content", subdescription: "Fields with the matching xml-content-name is assumed to be content.", input: "cat:\n +@name: tiger\n +content: cool\n", expected: "cool\n", scenarioType: "encode", }, { description: "round trip multiline 1", skipDoc: true, input: "\n", expected: "\n", scenarioType: "roundtrip", }, { description: "round trip multiline 2", skipDoc: true, input: "\n", expected: "\n", scenarioType: "roundtrip", }, { description: "round trip multiline 3", skipDoc: true, input: "\n", expected: "\n", scenarioType: "roundtrip", }, { description: "round trip multiline 4", skipDoc: true, input: "\n", expected: "\n", scenarioType: "roundtrip", }, { description: "round trip multiline 5", skipDoc: true, // pity spaces aren't kept atm. input: "\n", expected: "\n", scenarioType: "roundtrip", }, { description: "Encode xml: comments", subdescription: "A best attempt is made to copy comments to xml.", input: yamlWithComments, expected: expectedXMLWithComments, scenarioType: "encode", }, { description: "Encode: doctype and xml declaration", subdescription: "Use the special xml names to add/modify proc instructions and directives.", input: yamlWithProcInstAndDirectives, expected: expectedXmlWithProcInstAndDirectives, scenarioType: "encode", }, { description: "Round trip: with comments", subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.", input: inputXMLWithComments, expected: expectedRoundtripXMLWithComments, scenarioType: "roundtrip", }, { description: "Roundtrip: with doctype and declaration", subdescription: "yq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.", input: xmlWithProcInstAndDirectives, expected: expectedXmlWithProcInstAndDirectives, scenarioType: "roundtrip", }, } func testXMLScenario(t *testing.T, s formatScenario) { switch s.scenarioType { case "", "decode": yamlPrefs := ConfiguredYamlPreferences.Copy() yamlPrefs.Indent = 4 test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description) case "roundtrip": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description) case "decode-keep-ns": prefs := NewDefaultXmlPreferences() prefs.KeepNamespace = true test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "decode-raw-token": prefs := NewDefaultXmlPreferences() prefs.UseRawToken = true test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "decode-raw-token-off": prefs := NewDefaultXmlPreferences() prefs.UseRawToken = false test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) case "roundtrip-skip-directives": prefs := NewDefaultXmlPreferences() prefs.SkipDirectives = true test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description) case "decode-error": result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences)) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } case "encode-error": result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences())) if err == nil { t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result) } else { test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description) } default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return } switch s.scenarioType { case "", "decode": documentXMLDecodeScenario(w, s) case "encode": documentXMLEncodeScenario(w, s) case "roundtrip": documentXMLRoundTripScenario(w, s) case "decode-keep-ns": documentXMLDecodeKeepNsScenario(w, s) case "decode-raw-token-off": documentXMLDecodeKeepNsRawTokenScenario(w, s) case "roundtrip-skip-directives": documentXMLSkipDirectivesScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.xml file of:\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") expression := s.expression if s.expression != "" { expression = fmt.Sprintf(" '%v'", s.expression) } writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy%v sample.xml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences)))) } func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.xml file of:\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() prefs.KeepNamespace = false writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)))) prefsWithout := NewDefaultXmlPreferences() prefs.KeepNamespace = true writeOrPanic(w, "instead of\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout)))) } func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.xml file of:\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq --xml-raw-token=false sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() prefs.UseRawToken = false writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)))) prefsWithout := NewDefaultXmlPreferences() prefsWithout.UseRawToken = true writeOrPanic(w, "instead of\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout)))) } func documentXMLEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.yml file of:\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq -o=xml sample.yml\n```\n") writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)))) } func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.xml file of:\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq sample.xml\n```\n") writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)))) } func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { writeOrPanic(w, s.subdescription) writeOrPanic(w, "\n\n") } writeOrPanic(w, "Given a sample.xml file of:\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") writeOrPanic(w, "```bash\nyq --xml-skip-directives sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() prefs.SkipDirectives = true writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)))) } func TestXMLScenarios(t *testing.T) { for _, tt := range xmlScenarios { testXMLScenario(t, tt) } genericScenarios := make([]interface{}, len(xmlScenarios)) for i, s := range xmlScenarios { genericScenarios[i] = s } documentScenarios(t, "usage", "xml", genericScenarios, documentXMLScenario) } ================================================ FILE: pkg/yqlib/yaml.go ================================================ package yqlib type YamlPreferences struct { Indent int ColorsEnabled bool LeadingContentPreProcessing bool PrintDocSeparators bool UnwrapScalar bool EvaluateTogether bool FixMergeAnchorToSpec bool CompactSequenceIndent bool } func NewDefaultYamlPreferences() YamlPreferences { return YamlPreferences{ Indent: 2, ColorsEnabled: false, LeadingContentPreProcessing: true, PrintDocSeparators: true, UnwrapScalar: true, EvaluateTogether: false, FixMergeAnchorToSpec: false, CompactSequenceIndent: false, } } func (p *YamlPreferences) Copy() YamlPreferences { return YamlPreferences{ Indent: p.Indent, ColorsEnabled: p.ColorsEnabled, LeadingContentPreProcessing: p.LeadingContentPreProcessing, PrintDocSeparators: p.PrintDocSeparators, UnwrapScalar: p.UnwrapScalar, EvaluateTogether: p.EvaluateTogether, FixMergeAnchorToSpec: p.FixMergeAnchorToSpec, CompactSequenceIndent: p.CompactSequenceIndent, } } var ConfiguredYamlPreferences = NewDefaultYamlPreferences() ================================================ FILE: pkg/yqlib/yaml_test.go ================================================ package yqlib import ( "testing" "github.com/mikefarah/yq/v4/test" ) var yamlFormatScenarios = []formatScenario{ { description: "scalar with doc separator", skipDoc: true, input: "--- cat", expected: "---\ncat\n", }, { description: "CRLF doc separator", skipDoc: true, input: "---\r\ncat\r\n", expected: "---\r\ncat\r\n", }, { description: "yaml directive preserved (LF)", skipDoc: true, input: "%YAML 1.1\n---\ncat\n", expected: "%YAML 1.1\n---\ncat\n", }, { description: "yaml directive preserved (CRLF)", skipDoc: true, input: "%YAML 1.1\r\n---\r\ncat\r\n", expected: "%YAML 1.1\r\n---\r\ncat\r\n", }, { description: "comment only no trailing newline", skipDoc: true, input: "# hello", expected: "# hello\n", }, { description: "scalar with doc separator", skipDoc: true, input: "---cat", expected: "---cat\n", }, { description: "basic - null", skipDoc: true, input: "null", expected: "null\n", }, { description: "basic - ~", skipDoc: true, input: "~", expected: "~\n", }, { description: "octal", skipDoc: true, input: "0o30", expression: "tag", expected: "!!int\n", }, { description: "basic - [null]", skipDoc: true, input: "[null]", expected: "[null]\n", }, { description: "multi document anchor map", skipDoc: true, input: "a: &remember mike\n---\nb: *remember", expression: "explode(.)", expected: "a: mike\n---\nb: mike\n", }, { description: "basic - [~]", skipDoc: true, input: "[~]", expected: "[~]\n", }, { description: "basic - null map value", skipDoc: true, input: "a: null", expected: "a: null\n", }, { description: "basic - number", skipDoc: true, input: "3", expected: "3\n", }, { description: "basic - float", skipDoc: true, input: "3.1", expected: "3.1\n", }, { description: "basic - float", skipDoc: true, input: "[1, 2]", expected: "[1, 2]\n", }, } var yamlParseScenarios = []expressionScenario{ // { // description: "with a unquoted question mark in the string", // document: "foo: {bar: a?bc}", // expected: []string{ // "D0, P[], (!!map)::a: hello # things\n", // }, // }, { document: `a: hello # things`, expected: []string{ "D0, P[], (!!map)::a: hello # things\n", }, }, { document: "a: &a apple\nb: *a", expression: ".b | explode(.)", expected: []string{ "D0, P[b], (!!str)::apple\n", }, }, { document: `a: [1,2]`, expected: []string{ "D0, P[], (!!map)::a: [1, 2]\n", }, }, { document: `a: !horse [a]`, expected: []string{ "D0, P[], (!!map)::a: !horse [a]\n", }, }, } func testYamlScenario(t *testing.T, s formatScenario) { test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description) } func TestYamlParseScenarios(t *testing.T) { for _, tt := range yamlParseScenarios { testScenario(t, &tt) } } func TestYamlFormatScenarios(t *testing.T) { for _, tt := range yamlFormatScenarios { testYamlScenario(t, tt) } } ================================================ FILE: project-words.txt ================================================ abxbbxdbxebxczzx abxbbxdbxebxczzy accum Accum adithyasunil AEDT água ÁGUA alecthomas appleapple Astuff autocreating autoparse AWST axbxcxdxe axbxcxdxexxx bananabanana barp nbaz bitnami blarp blddir Bobo BODMAS bonapite Brien Bstuff BUILDKIT buildpackage catmeow CATYPE CBVVE chardata chillum choco chomper cleanup cmlu colorise colors Colors colourize compinit coolioo coverprofile createmap csvd CSVUTF currentlabel cygpath czvf datestring datetime Datetime datetimes DEBEMAIL debhelper Debugf debuild delish delpaths DELPATHS devorbitus devscripts dimchansky Dont dput elliotchance endhint endofname Entriesfrom envsubst errorlevel Escandón Evalall fakefilename fakeroot Farah fatih Fifi filebytes Fileish foobar foobaz foof frood fullpath gitbook githubactions gnupg goccy gofmt gogo golangci goreleaser GORELEASER GOMODCACHE GOPATH gosec gota goversion GOVERSION haha hellno herbygillot hexdump Hoang hostpath hotdog howdy incase inlinetables inplace ints ireduce iwatch jinzhu jq's jsond keygrip Keygrip KEYGRIP KEYID keyvalue kwak lalilu ldflags LDFLAGS lexer Lexer libdistro lindex linecomment LVAs magiconair mapvalues Mier mikefarah minideb minishift mipsle mitchellh mktemp Mult multidoc multimaint myenv myenvnonexisting myfile myformat ndjson NDJSON NFKD nixpkgs nojson nonascii nonempty noninteractive Nonquoting nosec notoml noxml nolua nullinput onea Oneshot opencollect opstack orderedmap osarch overridign pacman Padder pandoc parsechangelog pcsv pelletier pflag prechecking Prerelease proc propsd qylib readline realnames realpath repr rhash rindex risentveber rmescandon Rosey roundtrip roundtrips Roundtrip roundtripping Interp interp runningvms sadface selfupdate setpath sharedfolder Sharedfolder shellvariables shellvars shortfunc shortpipe shunit snapcraft somevalue splt srcdir stackoverflow stiched Strc strenv strload stylig subarray subchild subdescription submatch submatches SUBSTR tempfile tfstate Tfstate thar timezone Timezone timezones Timezones tojson Tokenvalue tsvd Tuan tzdata Uhoh updateassign urid utfbom Warningf Wazowski webi Webi wherever winget withdots wizz woop workdir Writable xmld xyzzy yamld yqlib yuin zabbix tonumber noyaml nolint shortfile Unmarshalling noini nocsv nobase64 nouri noprops nosh noshell tinygo nonexistent hclsyntax hclwrite nohcl zclconf cty go-cty Colorisation goimports errorlint RDBMS expeñded bananabananabananabanana edwinjhlee flox unlabelled kyaml KYAML nokyaml buildvcs behaviour GOFLAGS gocache subsubarray Ffile Fquery coverpkg gsub ================================================ FILE: release_instructions.txt ================================================ - update release notes - make sure local build passes - run ./scripts/copy-docs.sh (and commit the changed in the yq-book branch) - run ./scripts/bump-version.sh - git push // to move the v4 branch - git push origin -d v4 - git push --tags - use github actions to publish docker and make github release - check github updated yq action in marketplace - snapcraft - update snapcraft version - https://snapcraft.io/yq/builds - will auto create a candidate, test it works then promote - !! need to update v4/stable as well as latest sudo snap remove yq sudo snap install --edge yq then use the UI (https://snapcraft.io/yq/release) - brew - brew bump-formula-pr --url=https://github.com/mikefarah/yq/archive/2.2.0.tar.gz yq - if that fails with random ruby errors try: - clearing out the gems rm -rf .gem/ruby/2.3.0 - export HOMEBREW_FORCE_VENDOR_RUBY=1 - docker - build and push latest and new version tag - docker build . -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X - debian package (with release script) - execute the script `./scripts/release-deb.sh` *on the machine having the private gpg key to sign the generated sources* providing the version to release, the ppa where deploying the files and the passphrase of the private key if needed: ``` ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION ``` - debian package (manually) - ensure you get all vendor dependencies before packaging ```go mod vendor``` - execute ```dch -i``` - fill debian/changelog with changes from last version - build the package sources ```debuild -i -I -S -sa``` (signing with gpg key is required in order to put it to ppa) - put to PPA ```dput ppa: ../yq__source.changes``` (current distro repository is ppa:rmescandon/yq. In case that a new version is released, please contact rmescandon@gmail.com to bump debian package) ================================================ FILE: release_notes.txt ================================================ 4.52.4: - Dropping windows/arm - no longer supported in cross-compile 4.52.3: - Fixing comments in TOML arrays (#2592) - Bumped dependencies 4.52.2: - Fixed bad instructions file breaking go-install (#2587) Thanks @theyoprst - Fixed TOML table scope after comments (#2588) Thanks @tomers - Multiply uses a readonly context (#2558) - Fixed merge globbing wildcards in keys (#2564) - Fixing TOML subarray parsing issue (#2581) 4.52.1: - TOML encoder support - you can now roundtrip! #1364 - Parent now supports negative indices, and added a 'root' command for referencing the top level document - Fixed scalar encoding for HCL - Add --yaml-compact-seq-indent / -c flag for compact sequence indentation (#2583) Thanks @jfenal - Add symlink check to file rename util (#2576) Thanks @Elias-elastisys - Powershell fixed default command used for __completeNoDesc alias (#2568) Thanks @teejaded - Unwrap scalars in shell output mode. (#2548) Thanks @flintwinters - Added K8S KYAML output format support (#2560) Thanks @robbat2 - Bumped dependencies - Special shout out to @ccoVeille for reviewing my PRs! 4.50.1: - Added HCL support! - Fixing handling of CRLF #2352 - Bumped dependencies 4.49.2: - Fixing escape character bugs :sweat: #2517 - Fixing snap release pipeline #2518 Thanks @aalexjo 4.49.1: - Added `--security` flags to disable env and file ops #2515 - Fixing TOML ArrayTable parsing issues #1758 - Fixing parsing of escaped characters #2506 4.48.2: - Strip whitespace when decoding base64 #2507 - Upgraded to go-yaml v4! (thanks @ccoVeille, @ingydotnet) - Add linux/loong64 to release target (thanks @znley) - Added --shell-key-separator flag for customizable shell output format #2497 (thanks @rsleedbx) - Bumped dependencies 4.48.1: - Added 'parents' operator, to return a list of all the hierarchical parents of a node - Added 'first(exp)' operator, to return the first entry matching an expression in an array - Fixed xml namespace prefixes #1730 (thanks @baodrate) - Fixed out of range panic in yaml decoder #2460 (thanks @n471d) - Bumped dependencies 4.47.2: - Conversion from TOML to JSON no longer omits empty tables #2459 (thanks @louislouislouislouis) - Bumped dependencies 4.47.1: - Fixed merge anchor behaviour (`<<`); #2404, #2110, #2386, #2178 Huge thanks to @stevenwdv! Note that you will need to set --yaml-fix-merge-anchor-to-spec to see the fixes - Fixed panic for syntax error when creating a map #2423 - Bumped dependencies 4.46.1: - Added INI support - Fixed 'add' operator when piped in with no data #2378, #2383, #2384 - Fixed delete after slice problem (bad node path) #2387 Thanks @antoinedeschenes - Fixed yq small build Thanks @imzue - Switched to YAML org supported go-yaml! - Bumped dependencies 4.45.4: - Fixing wrong map() behaviour on empty map #2359 - Bumped dependencies 4.45.3: - Fixing regression introduced with in 4.45.2 with #2325 fix :sweat: sorry folks! - Bumped dependencies 4.45.2: - Added windows arm builds (Thanks @albertocavalcante, @ShukantPal) - Added s390x platform support (Thanks @ashokpariya0) - Additionally push docker images to ghcr.io (Thanks @reegnz) - Fixing add when there is no node match #2325 - sort_by works on maps - Bumped dependencies 4.45.1: - Create parent directories when --split-exp is used, Thanks @rudo-thomas - Bumped dependencies 4.44.6: - Fixed deleting items in array bug #2027, #2172; Thanks @jandubois - Docker image for armv7 / raspberry pi3, Thanks @brianegge - Fixed no-colors regression #2218 - Fixed various panic scenarios #2211 - Bumped dependencies 4.44.5: - Fixing release pipeline 4.44.4: - Format comments with a gray foreground (Thanks @gabe565) - Fixed handling of nulls with sort_by expressions #2164 - Force no color output when NO_COLOR env presents (Thanks @narqo) - Fixed array subtraction update bug #2159 - Fixed index out of range error - Can traverse straight from parent operator (parent.blah) - Bumped dependencies 4.44.3: - Fixed upper-case file extension detection, Thanks @ryenus (#2121) - Log printing follow no-colors flag #2082 - Skip and warn when interpolating strings and theres a unclosed bracket #2083 - Fixed CSV content starting with # issue #2076 - Bumped dependencies 4.44.2: - Handle numbers with underscores #2039 - Unique now works on maps and arrays #2068 - Added support for short hand splat with env[] expression #2071, as well as many other operators (split,select,eval,pick..) - Bumped dependencies 4.44.1: - Added min/max operators (#1992) Thanks @mbenson - Added pivot oeprator (#1993) Thanks @mbenson - Fix: shell-completion (#2006) Thanks @codekow - Handle escaped backslashes (#1997) Thanks @mbenson - Fix npe when given filename ending with "." (#1994) - Fix: linux (w/ selinux) build (#2004) Thanks @codekow - Bumped dependencies 4.43.1: - Added omit operator #1989 thanks @mbenson! - Added tostring #72 - Added string interpolation #1149 - Can specify parent(n) levels #1970 - Can now multiply strings by numbers #1988 thanks @mbenson! - Fixed CSV line break issue #1974 - Adding a EvaluateAll function to StringEvaluator #1966 - yqlib, default to colors off when using yaml library #1964 - Removed JSON output warning - Bumped dependencies 4.42.1: - Can execute yq expression files directly with shebang #1851 - Added --csv-separator flag #1950 - Added --properties-separator option - thanks @learnitall #1864 - Added --properties-array-brackets flag for properties encoder #1933 - Shell completion improvements - thanks @scop #1911 - Bumped dependencies 4.41.1: - Can now comment in yq expressions! #1919 - Fixed Toml decoding when table array defined before parent #1922 - Added new CSV option to turn off auto-parsing #1947 - Fixing with_entries context #1925 - Can now retrieve the alias names of merge anchors #1942 4.40.7: - Bumped dependencies 4.40.6: - Fix: empty TOML table #1924 - Thanks @elibroftw - Fixed "all" error message #1845 - Fixed to_entries[] - Bumped dependencies 4.40.5: - Fixing seg fault on bad XML #1888 - Fixed handling of --- #1890, #1896 - Bumped dependencies 4.40.4: - Fixed bug with creating maps with values based off keys #1886, #1889 - Bumped dependencies 4.40.3: - Fixed JSON output issue with empty arrays #1880 4.40.2: - Do not panic when StdIn is closed (#1867) Thanks @aleskandro! - Fixed issue when update against self #1869 - Fixed multi doc anchor bug #1861 - Fixes doc line separator issue when reading expression file #1860 - Bumped dependencies 4.40.1: - Added tonumber support (#1664, #71) - Added kind operator - Lua output fixes (#1811) - Thanks @Zash! - Add support for Lua input (#1810) - Thanks @Zash! - Rewrote parsing engine - yq now has its own AST! - Bumped dependencies 4.35.2: - Fix various typos #1798 - Fixed number parsing as float bug in JSON #1756 - Fixed string, null concatenation consistency #1712 - Fixed expression parsing issue #1711 - Bumped dependencies 4.35.1: - Added Lua output support (Thanks @Zash)! - Added BSD checksum format (Thanks @viq)! - Bumped dependencies 4.34.2: - Bumped dependencies 4.34.1: - Added shell output format thanks @giorgiga - Fixed nil pointer dereference (#1649) thanks @ArthurFritz - Bumped dependency versions 4.33.3: - Fixed bug when splatting empty array #1613 - Added scalar output for TOML (#1617) - Fixed passing of readonly context in pipe (partial fix for #1631) - Bumped dependency versions 4.33.2: - Add ``--nul-output|-0`` flag to separate element with NUL character (#1550) Thanks @vaab! - Add removable-media interface plug declaration to the snap packaging(#1618) Thanks @brlin-tw! - Scalar output now handled in csv, tsv and property files - Bumped dependency versions 4.33.1: - Added read-only TOML support! #1364. Thanks @pelletier for making your API available in your toml lib :) - Added warning when auto detect by file type is outputs JSON (#1608) 4.32.2: - Fixed behaviour for unknown file types (defaults to yaml) #1609 4.32.1: - Added divide and modulo operators (#1593) - thanks @teejaded! - Add support for decoding base64 strings without padding (#1555) - thanks @teejaded! - Add filter operation (#1588) - thanks @rbren! - Detect input format based on file name extension (#1582) - thanks @ryenus! - Auto output format when input format is automatically detected - Fixed npe in log #1596 - Improved binary file size! - Bumped dependency versions 4.31.2: - Fixed variable handling #1458, #1566 - Fixed merged anchor reference problem #1482 - Fixed xml encoding of ProcInst #1563, improved XML comment handling - Allow build without json and xml support (#1556) Thanks @afbjorklund - Bumped dependencies 4.31.1: - Added shuffle command #1503 - Added ability to sort by multiple fields #1541 - Added @sh encoder #1526 - Added @uri/@urid encoder/decoder #1529 - Fixed date comparison with string date #1537 - Added from_unix/to_unix Operators - Bumped dependency versions 4.30.8: - Log info message when unable to chown file in linux (e.g. snap confinement) #1521 4.30.7: - Fixed bug in splice operator #1511 - Fixed value operator bug #1515 - Fixed handling of merging null #1501 - Ownership of file now maintained in linux (thanks @vaguecoder) #1473 - Bumped dependency versions 4.30.6: - Fixed xml comment in array of scalars #1465 - Include blank newlines in leading header preprocessing #1462 - Added aarch64 build (#1261) - Bumped dependency versions (#1453) 4.30.5: - XML Decoder: Comment parsing tweak - XML Decoder: Fixed processing comments in empty XML #1446 - XML Decoder: Checking for invalid content outside of a root node #1448 - XML Decoder: Fixed issue where content surrounding tags are lost #1447 - XML Decoder: Fixed xml decode bug when there is content after a comment - Fixed loading yaml with header issue #1445 - guessTagFromCustomType warning log is now a debug. - Special thanks to @Kopfbremse for reporting XML issues! 4.30.4: - Fixed bug in automated versioning (snap/brew) 4.30.3: - Updated release process (automated versioning) - Fixed handling of yaml directives (#1424) - Fixed parsing of newline character in string expression #1430 - Fixed length compares to null instead of 0 issue #1427 4.30.2: - Actually updated the default xml prefix :facepalm: 4.30.1: - XML users note: the default attribute prefix has change to `+@` to avoid naming conflicts! - Can use expressions in slice #1419 - Fixed unhandled exception when decoding CSV thanks @washanhanzi - Added array_to_map operator for #1415 - Fixed sorting by date #1412 - Added check to ensure only maps can be encoded to XML #1408 - Check merge alias is a map #1425 - Explicitly setting unwrap flag works for json output #437, #1409 - Bumped go version 4.29.2: - Fixed null pointer exception when parsing CSV with empty field #1404 4.29.1: - Fixed Square brackets removing update #1342 - Added slice array operator (.[10:15]) #44 - XML decoder/encoder now parses directives and proc instructions (#1344). Please use the new skip flags [documented here](https://mikefarah.gitbook.io/yq/usage/xml) to ignore them. - XML users note that the default attribute prefix will change to `+@` in the 4.30 release to avoid naming conflicts! - Improved comment handling of decoders (breaking change for yqlib users sorry) - Fixed load operator bug when loading yaml file with multiple documents - Bumped Go compiler version - Bumped dependencies 4.28.2: - Fixed Github Actions issues (thanks @mattphelps-8451) - Fixed bug - can now delete documents #1377 - Fixed handling of UTF8 encoded CSVs #1373 - Detect and fail on missing closing brackets #1366 - yq Github actions now build docker image as part of release - Bumped dependencies 4.28.1: - Added `setpath` and `delpaths` operators, like jq (#1374) - Added `is_key` operator, to check if a match was a key when recursing - Added validation when attempting to add sequences to maps (#1341) 4.27.5: - Fixed relative merge bug #1333 4.27.4: - Fixed bug in alternative (//) operator, RHS being evaluated when it didn't need to be - Fixed footer comment issue #1231 - Github action now runs as root (as recommended by Github Actions doc) - Updated dependencies 4.27.3: - Added new 'c' merge and assign flag that clobbers custom tags - Bumped go dependency to fix CVE (#1316) - Updated dependencies 4.27.2: - Fixed JSON decoder to maintain object key order. 4.27.1: - Added 'json' decoder for support for multiple JSON documents in a single file (e.g. NDJSON) - Added 'csv' decoding, array of objects encoding, and round-triping - New StringEvaluator when using yq as a lib (thanks @leviliangtw) - Fixed XML decoding issue (#1284) 4.26.1: - Switched to new expression parser (#1264) - Don't clobber anchor when adding nodes (#1269) - New error operator for custom validation (#1259) - Added support for --wrapScalar=false in properties encoder (#1241) Thanks @dcarbone - Fix error on multiple assign (#1257) Thanks @care0717 - Bumped dependency versions 4.25.4: - Fixed panic when using multiply assign on multiple documents #1256 Thanks @care0717 4.25.3: - xml decoder now maintains namespaces by default. Use new flags to disable if required. Thanks @rndmit - Length and other similar operators no longer return comments (#1231) - When split expression includes an extension, dont add .yml automatically (#1165) - Map -r to --unwrapScalar to be more a drop in replacement for jq (#1245) Thanks @SuperSandro2000 - Fixing usage of quoted numeric keys #1247 - Bumped dependency versions 4.25.2: - Fixed comments disappearing from end of file (#1217) - Fixed empty base64 decoding error (#1209) - JSON output now in colors (#1208) - Added codeql and fixed minor issues - Bumped go-yaml library - Bumped go dependency 4.25.1: - Can specify a split expression file #1194 - Fixed append map bug when key matches value in existing map #1200 - Nicer error message when trying to use merge anchor tags other than maps #1184 - Fixed Don't automatically read stdin when the null input flag is used - Added type as an alias for tag #1195 - Fixes bug when using write in-place with no expression and multiple files #1193 4.24.5: - Fixed scenarios that dropped the first line if it's a comment (#1181) - Fixed updating existing empty map resulting in single line styling (#1176) - Fixed `with` operation bug (#1174) - Bumped go compiler 4.24.4: - Fixed docker release build 4.24.3: - Added from_props - Re-releasing, 4.24.2 release failed to publish correctly. 4.24.2: - Fixing release pipeline for go1.18 4.24.1: - Added comparison operators! (#94) - Bumped Golang to 1.18 (#1153) - XML parser no longer runs in strict mode (added new flag to run in strict mode) (#1155) 4.23.1: - Can now supply the envsubst operator with parameters (nounset, noempty, failfast). See [envsubst](https://mikefarah.gitbook.io/yq/operators/env-variable-operators) for details (#1137) - Bumped dependencies - Fixed '+=' problem with multiple matches #1145 - Fixed bug with "and", "or" evaluating the RHS when not needed - Fixed potential panic (thanks @mkatychev) - Tweaked CLI help (thanks @justin-f-perez) 4.22.1: - Added [pick] (https://mikefarah.gitbook.io/yq/operators/pick) operator - Can load expression from a file '--from-file' (#1120) - Fixed property auto expansion (#1127) 4.21.1: - Added [reverse](https://mikefarah.gitbook.io/yq/operators/reverse) operator - Added [string case](https://mikefarah.gitbook.io/yq/operators/string-operators) operators - Added [base64 support](https://mikefarah.gitbook.io/yq/operators/encode-decode) - Added [line](https://mikefarah.gitbook.io/yq/operators/line) and [column](https://mikefarah.gitbook.io/yq/operators/column) operators - Bumped dependency versions 4.20.2: - Fixed self assignment issue (#1107) - Fixed bad capture groups with multiple matches (#1114) - No longer auto-read from STDIN if there are files given (#1115) - Added missing load_props operator 4.20.1: - New [Date Operators](https://mikefarah.gitbook.io/yq/operators/datetime) (now, tz, add and subtract durations from dates) - Can now decode property files! - New flag to manually set expression if required - ZSH completion bug fix (#1108) thanks @whi-tw - Fixed SEGV error (#1096) - Fixed Github actions issues (it pipes in /dev/null) for XML - Fixed bug - handle expressions that match a directory (e.g. ".") 4.19.1: - New [eval](https://mikefarah.gitbook.io/yq/operators/eval) _operator_ that allows dynamic expression evaluation (e.g. from an env variable) (#1087) - Adding new elements to array now automatically applies styling of existing elements (#722) 4.18.1: - `eval` is now the _default_ command, you can leave it out #113 - `-` no longer needs to be specified as STDIN, unless you are also working with multiple files. #113 - Adding to empty maps / arrays now uses idiomatic yaml styling by default - Fixed seg fault on bad input #1086 - New `envsubst` operator! (thanks @sciyoshi) - Added support for `*=`, relative multiply/merge - Custom tag types now autocast to there actual types #933 4.17.2: - Fixed manpath issue (thanks @mr-pmillz) 4.17.1: - Added XML support (#491) - New merge flag (n) to only merge new fields (#1038) - Fixed exit status bug for permission denied error (#1062) - Fixed using multiple variables with union (,) operator (#1048) - Bumped some versions of dependencies 4.16.2: - Bumped go-lang compiler to fix CVE-2021-44717 (#1037) - Dependency version bumps via dependabot - Added extract-checksum.sh to make it easier to validate checksums (#1011) - Report filename on parsing error (#1030) 4.16.1: - Added csv, tsv output formats - Added map, map_values operators - Added sort, sort_by operators (#947, #1024) - Fixed bug in collect - Fixed permissions issue in Dockerfile (#1014) - Fixed assignment operator to no longer overwrite anchor (#1029) 4.15.1: - Added 'load/strload' operators for dynamically loading content from files - Added 'key' operator - Added 'parent' operator - Smarter MAN page installation script (thanks @coolaj86) - Dockerfile improvements (thanks @actualben) - Error handling improvements (thanks @mmorel-35) 4.14.2: - Fixed header preprocessing issue (#1000) - Bumped version dependencies 4.14.1: - Added group_by operator - Added encode/decode operators (toyaml, fromjson etc) (#974) - Added flatten operator - Added --split-exp, for splitting results into multiple files (#966) - Fixed json null array bug (#985) 4.13.5: - Performance improvement for deepMatch (thanks @pmatseykanets) - Added manpage, included in tar.gz downloads as well as a separate tar.gz (#961) - Fixed expression parsing bug #970 - Rebuild fixes CVE (#964) - Bumped docker alpine version 4.13.4: - Fixed select bug (#958) - Improved performance of `explode` (this will also speed up json conversion) - Improved performance of `merge` (significantly if your merging a small file into a big one) 4.13.3: - Updated go compiler to 1.17 to fix CVE (#944) 4.13.2: - Fixing Docker build timeout issues when attempting to release 4.13.1: - Update to `with` operator, allow for no leading space on the `;`. 4.13.0: BREAKING CHANGE - the `as` variable operator (e.g. `.a as $x`) now makes a _copy_ of the node(s) at the path rather than a reference. This is in order to make it work more like the `jq` equivalent. This means any updates made to that variable do not update the original. There's a new operator `ref` that will make a reference (and allow multiple updates to the original path by referencing the variable). Sorry for any inconvenience caused!. - New `with` operator for making multiple changes to a given path - New `contains` operator, works like the `jq` equivalent - Subtract operator now supports subtracting elements from arrays! - Fixed Swapping values using variables #934 - Github Action now properly supports multiline output #936, thanks @pjxiao - Fixed missing closing bracket validation #932 4.12.2: - Fix processing of hex numbers #929 - Fixed alternative and union operator issues #930 4.12.1: - Merge comment fix #919 4.12.0: - Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info. - Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919) - pretty print yaml 1.1 compatibility (https://github.com/mikefarah/yq/issues/914) ================================================ FILE: scripts/acceptance.sh ================================================ #! /bin/bash set -e for test in acceptance_tests/*.sh; do echo "--------------------------------------------------------------" echo "$test" echo "--------------------------------------------------------------" (exec "$test"); done ================================================ FILE: scripts/build-small-yq.sh ================================================ #!/bin/bash go build -tags "yq_nolua yq_noini yq_notoml yq_noxml yq_nojson yq_nohcl yq_nokyaml" -ldflags "-s -w" . ================================================ FILE: scripts/build-tinygo-yq.sh ================================================ #!/bin/bash # Currently, the `yq_nojson` feature must be enabled when using TinyGo. tinygo build -no-debug -tags "yq_nolua yq_noini yq_notoml yq_noxml yq_nojson yq_nocsv yq_nobase64 yq_nouri yq_noprops yq_nosh yq_noshell yq_nohcl yq_nokyaml" . ================================================ FILE: scripts/bump-version.sh ================================================ #!/bin/bash set -e if [ "$1" == "" ]; then echo "Please specify at a version" exit 1 fi version=$1 # validate version is in the right format echo $version | sed -r '/v4\.[0-9][0-9]\.[0-9][0-9]?$/!{q1}' previousVersion=$(cat cmd/version.go| sed -n 's/.*Version = "\([^"]*\)"/\1/p') echo "Updating from $previousVersion to $version" sed -i "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go go build . actualVersion=$(./yq --version) if [ "$actualVersion" != "yq (https://github.com/mikefarah/yq/) version $version" ]; then echo "Failed to update version.go" exit 1 else echo "version.go updated" fi version=$version ./yq -i '.version=strenv(version) | .parts.yq.source-tag=strenv(version)' snap/snapcraft.yaml actualSnapVersion=$(./yq '.version' snap/snapcraft.yaml) if [ "$actualSnapVersion" != "$version" ]; then echo "Failed to update snapcraft" exit 1 else echo "snapcraft updated" fi actualSnapVersion=$(./yq '.parts.yq.source-tag' snap/snapcraft.yaml) if [ "$actualSnapVersion" != "$version" ]; then echo "Failed to update snapcraft" exit 1 else echo "snapcraft updated" fi git add cmd/version.go snap/snapcraft.yaml git commit -m 'Bumping version' git tag $version git tag -f v4 ================================================ FILE: scripts/check.sh ================================================ #!/bin/bash set -o errexit set -o pipefail # TODO: Check if the found golangci-lint version matches the expected version (e.g., v1.62.0), especially if falling back to PATH version. GOPATH_LINT="$(go env GOPATH)/bin/golangci-lint" BIN_LINT="./bin/golangci-lint" LINT_CMD="" if [ -f "$GOPATH_LINT" ]; then LINT_CMD="$GOPATH_LINT" elif [ -f "$BIN_LINT" ]; then LINT_CMD="$BIN_LINT" elif command -v golangci-lint &> /dev/null; then # Using PATH version, ensure compatibility (see TODO) LINT_CMD="golangci-lint" else echo "Error: golangci-lint not found in $GOPATH/bin, ./bin, or PATH." echo "Please run scripts/devtools.sh or ensure golangci-lint is installed correctly." exit 1 fi GOFLAGS="${GOFLAGS}" "$LINT_CMD" run --verbose ================================================ FILE: scripts/compare-jq.sh ================================================ #!/usr/bin/env bash set -e exp=$1 file=$2 if [ "$2" == "" ]; then echo "yq" ./yq -oj -n "$1" echo "jq" jq -n "$1" else echo "yq" ./yq -oj "$1" $2 echo "jq" ./yq $2 -oj | jq "$1" fi ================================================ FILE: scripts/compare-versions-output.sh ================================================ #!/usr/bin/env bash test_data=' - foo: false ' for version in 4.45.1 4.45.2 4.45.3; do for command in '.[] | (select(.foo) | {"foo": .foo} // {})' '.[] | (select(.foo) | {.foo} // {})'; do echo ${version} "${command}" echo ------- echo "${test_data}" | podman run -i --rm mikefarah/yq:${version} -o json "${command}" echo ------- echo done done ================================================ FILE: scripts/copy-docs.sh ================================================ #!/bin/bash cp how-it-works.md ../yq-gitbook/. cp pkg/yqlib/doc/operators/*.md ../yq-gitbook/operators/. cp pkg/yqlib/doc/usage/*.md ../yq-gitbook/usage/. ================================================ FILE: scripts/coverage.sh ================================================ #!/bin/bash set -e echo "Running tests and generating coverage..." packages=$(go list ./... | grep -v -E 'examples' | grep -v -E 'test' | tr '\n' ',' | sed 's/,$//') test_packages=$(go list ./... | grep -v -E 'examples' | grep -v -E 'test' | grep -v '^github.com/mikefarah/yq/v4$') go test -coverprofile=coverage.out -coverpkg="$packages" -v $test_packages echo "Generating HTML coverage report..." go tool cover -html=coverage.out -o coverage.html echo "" echo "Generating sorted coverage table..." # Create a simple approach using grep and sed to extract file coverage # First, get the total coverage total_coverage=$(go tool cover -func=coverage.out | grep "^total:" | sed 's/.*([^)]*)[[:space:]]*\([0-9.]*\)%.*/\1/') # Extract file-level coverage by finding the last occurrence of each file go tool cover -func=coverage.out | grep -E "\.go:[0-9]+:" | \ sed 's/^\([^:]*\.go\):.*[[:space:]]\([0-9.]*\)%.*/\2 \1/' | \ sort -k2 | \ awk '{file_coverage[$2] = $1} END {for (file in file_coverage) printf "%.2f %s\n", file_coverage[file], file}' | \ sort -nr > coverage_sorted.txt # Add total coverage to the file if [[ -n "$total_coverage" && "$total_coverage" != "0" ]]; then echo "TOTAL: $total_coverage" >> coverage_sorted.txt fi echo "" echo "Coverage Summary (sorted by percentage - lowest coverage first):" echo "=================================================================" printf "%-60s %10s %12s\n" "FILE" "COVERAGE" "STATUS" echo "=================================================================" # Display results with status indicators tail -n +1 coverage_sorted.txt | while read percent file; do if [[ "$file" == "TOTAL:" ]]; then echo "" printf "%-60s %8s%% %12s\n" "OVERALL PROJECT COVERAGE" "$percent" "📊 TOTAL" echo "=================================================================" continue fi filename=$(basename "$file") status="" if (( $(echo "$percent < 50" | bc -l 2>/dev/null || echo "0") )); then status="🔴 CRITICAL" elif (( $(echo "$percent < 70" | bc -l 2>/dev/null || echo "0") )); then status="🟡 LOW" elif (( $(echo "$percent < 90" | bc -l 2>/dev/null || echo "0") )); then status="🟢 GOOD" else status="✅ EXCELLENT" fi printf "%-60s %8s%% %12s\n" "$filename" "$percent" "$status" done echo "" echo "Top 10 files by uncovered statements:" echo "=================================================" # Calculate uncovered statements for each file and sort by that go tool cover -func=coverage.out | grep -E "\.go:[0-9]+:" | \ awk '{ # Extract filename and percentage split($1, parts, ":") file = parts[1] pct = $NF gsub(/%/, "", pct) # Track stats per file total[file]++ covered[file] += pct } END { for (file in total) { avg_pct = covered[file] / total[file] uncovered = total[file] * (100 - avg_pct) / 100 covered_count = total[file] - uncovered printf "%.0f %d %.0f %.1f %s\n", uncovered, total[file], covered_count, avg_pct, file } }' | sort -rn | head -10 | while read uncovered total covered pct file; do filename=$(basename "$file") printf "%-60s %4d uncovered (%4d/%4d, %5.1f%%)\n" "$filename" "$uncovered" "$covered" "$total" "$pct" done echo "" echo "Coverage reports generated:" echo "- HTML report: coverage.html (detailed line-by-line coverage)" echo "- Sorted table: coverage_sorted.txt" echo "- Use 'go tool cover -func=coverage.out' for function-level details" ================================================ FILE: scripts/devtools.sh ================================================ #!/bin/sh set -ex go mod download golang.org/x/tools@latest curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.5 curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11 ================================================ FILE: scripts/extract-checksum.sh ================================================ #!/bin/bash # This script works with checksums_hashes_order and checksums to extract the relevant # sha of the various yq downloads. You can then use your favourite checksum tool to validate. # must match an entry in checksums_hashes_order. # # Usage: ./extract-checksum.sh # E.g: ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz # Outputs: # yq_linux_amd64.tar.gz acebc9d07aa2d0e482969b2c080ee306e8f58efbd6f2d857eefbce6469da1473 # # Usage with rhash: # ./extract-checksum.sh SHA-256 yq_linux_amd64.tar.gz | rhash -c - # # Tip, if you want the checksum first then the filename (e.g. for the md5sum command) # then you can pipe the output of this script into awk to switch the fields around: # # ./extract-checksum.sh MD5 yq_linux_amd64.tar.gz | awk '{ print $2 " " $1}' | md5sum -c - # # if [ "$1" == "" ]; then echo "Please specify at a hash algorithm from the checksum_hashes_order" echo "Usage: $0 " exit 1 fi if [ "$2" != "" ]; then # so we don't match x.tar.gz when 'x' is given file="$2\s" else file="" fi if [ ! -f "checksums_hashes_order" ]; then echo "This script requires checksums_hashes_order to run" echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate" exit 1 fi if [ ! -f "checksums" ]; then echo "This script requires the checksums file to run" echo "Download the file from https://github.com/mikefarah/yq/releases/ for the version of yq you are trying to validate" exit 1 fi grepMatch=$(grep -m 1 -n "$1" checksums_hashes_order) if [ "$grepMatch" == "" ]; then echo "Could not find hash algorithm '$1' in checksums_hashes_order" exit 1 fi set -e lineNumber=$(echo "$grepMatch" | cut -f1 -d:) realLineNumber="$(($lineNumber + 1))" grep "$file" checksums | sed 's/ /\t/g' | cut -f1,$realLineNumber ================================================ FILE: scripts/format.sh ================================================ #!/bin/bash gofmt -w -s . go mod tidy go mod vendor ================================================ FILE: scripts/generate-man-page-md.sh ================================================ #! /bin/bash set -e # note that this requires pandoc to be installed. cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md printf "\n# HOW IT WORKS\n" >> man.md tail -n +2 how-it-works.md >> man.md for f in ./pkg/yqlib/doc/operators/*.md; do cat "$f" >> man.md done for f in ./pkg/yqlib/doc/usage/*.md; do cat "$f" >> man.md done ================================================ FILE: scripts/generate-man-page.sh ================================================ #! /bin/bash set -e # note that this requires pandoc to be installed. pandoc \ --variable=title:"YQ" \ --variable=section:"1" \ --variable=author:"Mike Farah" \ --variable=header:"${MAN_HEADER}" \ --standalone --to man man.md -o yq.1 ================================================ FILE: scripts/install-man-page.sh ================================================ #!/bin/sh my_path="$(command -v yq)" if [ -z "$my_path" ]; then echo "'yq' wasn't found in your PATH, so we don't know where to put the man pages." echo "Please update your PATH to include yq, and run this script again." exit 1 fi # ex: ~/.local/bin/yq => ~/.local/ my_prefix="$(dirname "$(dirname "$(command -v yq)")")" mkdir -p "$my_prefix/share/man/man1/" cp yq.1 "$my_prefix/share/man/man1/" ================================================ FILE: scripts/release-deb.sh ================================================ #!/bin/bash -eux # # Copyright (C) 2021 Roberto Mier Escandón # # This script creates a .deb package file with yq valid for ubuntu 20.04 by default # You can pass DOCKER_IMAGE_NAME=yq-deb-builder DOCKER_IMAGE_TAG=$(git describe --always --tags) OUTPUT= GOVERSION="1.17.4" KEYID= MAINTAINER= DO_PUBLISH= PPA="rmescandon/yq" VERSION= DISTRIBUTION= DO_SIGN= PASSPHRASE= show_help() { echo " usage: $(basename "$0") VERSION [options...]" echo "" echo " positional arguments" echo " VERSION" echo "" echo " optional arguments:" echo " -h, --help Shows this help" echo " -d, --distribution DISTRO The distribution to use for the changelog generation. If not provided, last changelog entry" echo " distribution is considered" echo " --goversion VERSION The version of Golang to use. Default to $GOVERSION" echo " -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this" echo " parameter, the generated sources are not signed" echo " -s, --sign Sign the package sources with a gpg key of the maintainer" echo " -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is" echo " the considered format" echo " -o DIR, --output DIR The path where leaving the generated debian package. Default to a temporary folder if not set" echo " -p The resultant file is being published to ppa" echo " --ppa PPA Push resultant files to indicated ppa. This option should be given along with a signing key." echo " Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'" echo " --passphrase PASSPHRASE Passphrase to decrypt the signage key" exit 1 } # read input args while [ $# -ne 0 ]; do case $1 in -h|--help) show_help ;; -d|--distribution) shift DISTRIBUTION="$1" ;; --goversion) shift GOVERSION="$1" ;; -k|--sign-key) shift DO_SIGN='y' KEYID="$1" ;; -s|--sign) DO_SIGN='y' ;; -m|--maintainer) shift MAINTAINER="$1" ;; -o|--output) shift OUTPUT="$1" ;; -p) DO_PUBLISH="y" ;; --ppa) shift DO_PUBLISH="y" PPA="$1" ;; --passphrase) shift PASSPHRASE="$1" ;; *) if [ -z "$VERSION" ]; then VERSION="$1" else show_help fi esac shift done [ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help) if [ -n "$OUTPUT" ]; then OUTPUT="$(realpath "$OUTPUT")" mkdir -p "$OUTPUT" else # Temporary folder where leaving the built deb package in case that output folder is not provided OUTPUT="$(mktemp -d)" fi # Define the folders with the source project and the build artifacts and files srcdir="$(realpath "$(dirname "$0")"/..)" blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")" # clean on exit cleanup() { rm -f "${blddir}/build.sh" || true rm -f "${blddir}/Dockerfile" || true rm -f "${blddir}/dput.cf" || true docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true } trap cleanup EXIT INT # configure the dput config in case publishing is requested lp_id="$(echo "$PPA" | cut -d'/' -f1)" ppa_id="$(echo "$PPA" | cut -d'/' -f2)" cat << EOF > ${blddir}/dput.cf [ppa] fqdn = ppa.launchpad.net method = ftp incoming = ~${lp_id}/ubuntu/${ppa_id} login = anonymous EOF # create the main script cat << EOF > ${blddir}/build.sh #!/bin/bash set -e -o pipefail PATH=$PATH:/usr/local/go/bin export GPG_TTY=$(tty) go mod vendor ### bump debian/changelog # maintainer export DEBEMAIL="$MAINTAINER" if [ -z "$MAINTAINER" ]; then export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)" fi # prepend a 'v' char to complete the tag name from where calculating the changelog SINCE="v\$(dpkg-parsechangelog -S Version)" # distribution DISTRIBUTION="$DISTRIBUTION" if [ -z "$DISTRIBUTION" ]; then DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)" fi # generate changelog gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION" # using -d to prevent failing when searching for Golang dep on control file params=("-d" "-S") # add the -sa option for signing along with the key to use when provided key id if [ -n "$DO_SIGN" ]; then params+=("-sa") # read from gpg the key id associated with the maintainer if not provided explicitly if [ -z "$KEYID" ]; then KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)" else KEYID="$KEYID" fi params+=("--sign-key="\$KEYID"") if [ -n "$PASSPHRASE" ]; then gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000 KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)" /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP" fi else params+=("-us" "-uc") fi debuild \${params[@]} mv ../yq_* /home/yq/output echo "" echo -e "\tfind resulting package at: "$OUTPUT"" # publish to ppa whether given if [ -n "$DO_PUBLISH" ]; then dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes fi EOF chmod +x "${blddir}"/build.sh # build the docker image with all dependencies cat << EOF > ${blddir}/Dockerfile FROM bitnami/minideb:bullseye as base ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV DEBIAN_FRONTEND noninteractive ENV GO111MODULE on ENV GOMODCACHE /home/yq/go RUN set -e \ && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \ && apt-get -qq update # install Golang on its $GOVERSION FROM base as golang RUN apt-get -qq -y --no-install-recommends install \ ca-certificates \ wget RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4 RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz" FROM base RUN apt-get -qq -y --no-install-recommends install \ build-essential \ debhelper \ devscripts \ dput \ fakeroot \ git-buildpackage \ gpg-agent \ libdistro-info-perl \ pandoc \ rsync \ sensible-utils && \ apt-get clean && \ rm -rf /tmp/* /var/tmp/* COPY --from=golang /usr/local/go /usr/local/go # build debian package as yq user RUN useradd -ms /bin/bash yq && \ mkdir /home/yq/src && chown -R yq: /home/yq/src && \ mkdir /home/yq/output && chown -R yq: /home/yq/output ADD ./build/dput.cf /etc/dput.cf ADD ./build/build.sh /usr/bin/build.sh RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh USER yq WORKDIR /home/yq/src VOLUME ["/home/yq/src"] # dir where output packages are finally left VOLUME ["/home/yq/output"] CMD ["/usr/bin/build.sh"] EOF DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" . docker run --rm -i \ -v "${srcdir}":/home/yq/src:delegated \ -v "${OUTPUT}":/home/yq/output \ -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \ -u "$(id -u)" \ "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" ================================================ FILE: scripts/secure.sh ================================================ #!/bin/bash set -o errexit set -o pipefail OPTS=( -exclude-dir=vendor -exclude-dir=.gomodcache -exclude-dir=.gocache ) command -v gosec &> /dev/null && BIN=gosec || BIN=./bin/gosec "${BIN}" "${OPTS[@]}" "${PWD}" ./... ================================================ FILE: scripts/setup.sh ================================================ #!/bin/bash set -eu find_mgr() { if hash minishift 2>/dev/null; then echo "minishift" else if hash docker-machine 2>/dev/null; then echo "docker-machine" fi fi } get_vm_name() { case "$1" in minishift) echo "minishift" ;; docker-machine) echo "${DOCKER_MACHINE_NAME}" ;; *) ;; esac } is_vm_running() { local vm=$1 declare -a running=($(VBoxManage list runningvms | awk '{ print $1 }')) local result='false' for rvm in "${running[@]}"; do if [[ "${rvm}" == *"${vm}"* ]]; then result='true' fi done echo "$result" } if hash cygpath 2>/dev/null; then PROJECT_DIR=$(cygpath -w -a "$(pwd)") else PROJECT_DIR=$(pwd) fi VM_MGR=$(find_mgr) if [[ -z $VM_MGR ]]; then echo "ERROR: No VM Manager found; expected one of ['minishift', 'docker-machine']" exit 1 fi VM_NAME=$(get_vm_name "$VM_MGR") if [[ -z $VM_NAME ]]; then echo "ERROR: No VM found; try running 'eval $(docker-machine env)'" exit 1 fi if ! hash VBoxManage 2>/dev/null; then echo "VirtualBox executable 'VBoxManage' not found in path" exit 1 fi avail=$(is_vm_running "$VM_NAME") if [[ "$avail" == *"true"* ]]; then res=$(VBoxManage sharedfolder add "${VM_NAME}" --name "${PROJECT}" --hostpath "${PROJECT_DIR}" --transient 2>&1) if [[ -z $res || $res == *"already exists"* ]]; then # no need to show that it already exists : else echo "$res" exit 1 fi echo "VM: [${VM_NAME}] -- Added Sharedfolder [${PROJECT}] @Path [${PROJECT_DIR}]" else echo "$VM_NAME is not currently running; please start your VM and try again." exit 1 fi SSH_CMD="sudo mkdir -p /${PROJECT} ; sudo mount -t vboxsf ${PROJECT} /${PROJECT}" case "${VM_MGR}" in minishift) minishift ssh "${SSH_CMD}" echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]" ;; docker-machine) docker-machine ssh "${VM_NAME}" "${SSH_CMD}" echo "VM: [${VM_NAME}] -- Mounted Sharedfolder [${PROJECT}] @VM Path [/${PROJECT}]" ;; *) ;; esac ================================================ FILE: scripts/shunit2 ================================================ #! /bin/sh # vim:et:ft=sh:sts=2:sw=2 # # Copyright 2008-2020 Kate Ward. All Rights Reserved. # Released under the Apache 2.0 license. # http://www.apache.org/licenses/LICENSE-2.0 # # shUnit2 -- Unit testing framework for Unix shell scripts. # https://github.com/kward/shunit2 # # Author: kate.ward@forestent.com (Kate Ward) # # shUnit2 is a xUnit based unit test framework for Bourne shell scripts. It is # based on the popular JUnit unit testing framework for Java. # # $() are not fully portable (POSIX != portable). # shellcheck disable=SC2006 # expr may be antiquated, but it is the only solution in some cases. # shellcheck disable=SC2003 # Return if shunit2 already loaded. command [ -n "${SHUNIT_VERSION:-}" ] && exit 0 SHUNIT_VERSION='2.1.8' # Return values that scripts can use. SHUNIT_TRUE=0 SHUNIT_FALSE=1 SHUNIT_ERROR=2 # Logging functions. _shunit_warn() { ${__SHUNIT_CMD_ECHO_ESC} \ "${__shunit_ansi_yellow}shunit2:WARN${__shunit_ansi_none} $*" >&2 } _shunit_error() { ${__SHUNIT_CMD_ECHO_ESC} \ "${__shunit_ansi_red}shunit2:ERROR${__shunit_ansi_none} $*" >&2 } _shunit_fatal() { ${__SHUNIT_CMD_ECHO_ESC} \ "${__shunit_ansi_red}shunit2:FATAL${__shunit_ansi_none} $*" >&2 exit ${SHUNIT_ERROR} } # Determine some reasonable command defaults. __SHUNIT_CMD_ECHO_ESC='echo -e' # shellcheck disable=SC2039 command [ "`echo -e test`" = '-e test' ] && __SHUNIT_CMD_ECHO_ESC='echo' __SHUNIT_UNAME_S=`uname -s` case "${__SHUNIT_UNAME_S}" in BSD) __SHUNIT_CMD_EXPR='gexpr' ;; *) __SHUNIT_CMD_EXPR='expr' ;; esac __SHUNIT_CMD_TPUT='tput' # Commands a user can override if needed. SHUNIT_CMD_EXPR=${SHUNIT_CMD_EXPR:-${__SHUNIT_CMD_EXPR}} SHUNIT_CMD_TPUT=${SHUNIT_CMD_TPUT:-${__SHUNIT_CMD_TPUT}} # Enable color output. Options are 'never', 'always', or 'auto'. SHUNIT_COLOR=${SHUNIT_COLOR:-auto} # Specific shell checks. if command [ -n "${ZSH_VERSION:-}" ]; then setopt |grep "^shwordsplit$" >/dev/null if command [ $? -ne ${SHUNIT_TRUE} ]; then _shunit_fatal 'zsh shwordsplit option is required for proper operation' fi if command [ -z "${SHUNIT_PARENT:-}" ]; then _shunit_fatal "zsh does not pass \$0 through properly. please declare \ \"SHUNIT_PARENT=\$0\" before calling shUnit2" fi fi # # Constants # __SHUNIT_MODE_SOURCED='sourced' __SHUNIT_MODE_STANDALONE='standalone' __SHUNIT_PARENT=${SHUNIT_PARENT:-$0} # User provided test prefix to display in front of the name of the test being # executed. Define by setting the SHUNIT_TEST_PREFIX variable. __SHUNIT_TEST_PREFIX=${SHUNIT_TEST_PREFIX:-} # ANSI colors. __SHUNIT_ANSI_NONE='\033[0m' __SHUNIT_ANSI_RED='\033[1;31m' __SHUNIT_ANSI_GREEN='\033[1;32m' __SHUNIT_ANSI_YELLOW='\033[1;33m' __SHUNIT_ANSI_CYAN='\033[1;36m' # Set the constants readonly. __shunit_constants=`set |grep '^__SHUNIT_' |cut -d= -f1` echo "${__shunit_constants}" |grep '^Binary file' >/dev/null && \ __shunit_constants=`set |grep -a '^__SHUNIT_' |cut -d= -f1` for __shunit_const in ${__shunit_constants}; do if command [ -z "${ZSH_VERSION:-}" ]; then readonly "${__shunit_const}" else case ${ZSH_VERSION} in [123].*) readonly "${__shunit_const}" ;; *) readonly -g "${__shunit_const}" # Declare readonly constants globally. esac fi done unset __shunit_const __shunit_constants # # Internal variables. # # Variables. __shunit_lineno='' # Line number of executed test. __shunit_mode=${__SHUNIT_MODE_SOURCED} # Operating mode. __shunit_reportGenerated=${SHUNIT_FALSE} # Is report generated. __shunit_script='' # Filename of unittest script (standalone mode). __shunit_skip=${SHUNIT_FALSE} # Is skipping enabled. __shunit_suite='' # Suite of tests to execute. __shunit_clean=${SHUNIT_FALSE} # _shunit_cleanup() was already called. # ANSI colors (populated by _shunit_configureColor()). __shunit_ansi_none='' __shunit_ansi_red='' __shunit_ansi_green='' __shunit_ansi_yellow='' __shunit_ansi_cyan='' # Counts of tests. __shunit_testSuccess=${SHUNIT_TRUE} __shunit_testsTotal=0 __shunit_testsPassed=0 __shunit_testsFailed=0 # Counts of asserts. __shunit_assertsTotal=0 __shunit_assertsPassed=0 __shunit_assertsFailed=0 __shunit_assertsSkipped=0 # # Macros. # # shellcheck disable=SC2016,SC2089 _SHUNIT_LINENO_='eval __shunit_lineno=""; if command [ "${1:-}" = "--lineno" ]; then command [ -n "$2" ] && __shunit_lineno="[$2] "; shift 2; fi' #----------------------------------------------------------------------------- # Assertion functions. # # Assert that two values are equal to one another. # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertEquals() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertEquals() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_expected_=$1 shunit_actual_=$2 shunit_return=${SHUNIT_TRUE} if command [ "${shunit_expected_}" = "${shunit_actual_}" ]; then _shunit_assertPass else failNotEquals "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" shunit_return=${SHUNIT_FALSE} fi unset shunit_message_ shunit_expected_ shunit_actual_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_EQUALS_='eval assertEquals --lineno "${LINENO:-}"' # Assert that two values are not equal to one another. # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertNotEquals() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertNotEquals() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_expected_=$1 shunit_actual_=$2 shunit_return=${SHUNIT_TRUE} if command [ "${shunit_expected_}" != "${shunit_actual_}" ]; then _shunit_assertPass else failSame "${shunit_message_}" "${shunit_expected_}" "${shunit_actual_}" shunit_return=${SHUNIT_FALSE} fi unset shunit_message_ shunit_expected_ shunit_actual_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_NOT_EQUALS_='eval assertNotEquals --lineno "${LINENO:-}"' # Assert that a container contains a content. # # Args: # message: string: failure message [optional] # container: string: container to analyze # content: string: content to find # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertContains() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertContains() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_container_=$1 shunit_content_=$2 shunit_return=${SHUNIT_TRUE} if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then _shunit_assertPass else failNotFound "${shunit_message_}" "${shunit_content_}" shunit_return=${SHUNIT_FALSE} fi unset shunit_message_ shunit_container_ shunit_content_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_CONTAINS_='eval assertContains --lineno "${LINENO:-}"' # Assert that a container does not contain a content. # # Args: # message: string: failure message [optional] # container: string: container to analyze # content: string: content to look for # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertNotContains() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertNotContains() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_container_=$1 shunit_content_=$2 shunit_return=${SHUNIT_TRUE} if echo "$shunit_container_" | grep -F -- "$shunit_content_" > /dev/null; then failFound "${shunit_message_}" "${shunit_content_}" shunit_return=${SHUNIT_FALSE} else _shunit_assertPass fi unset shunit_message_ shunit_container_ shunit_content_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_NOT_CONTAINS_='eval assertNotContains --lineno "${LINENO:-}"' # Assert that a value is null (i.e. an empty string) # # Args: # message: string: failure message [optional] # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertNull() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 1 -o $# -gt 2 ]; then _shunit_error "assertNull() requires one or two arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi assertTrue "${shunit_message_}" "[ -z '$1' ]" shunit_return=$? unset shunit_message_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_NULL_='eval assertNull --lineno "${LINENO:-}"' # Assert that a value is not null (i.e. a non-empty string) # # Args: # message: string: failure message [optional] # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertNotNull() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -gt 2 ]; then # allowing 0 arguments as $1 might actually be null _shunit_error "assertNotNull() requires one or two arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_actual_=`_shunit_escapeCharactersInString "${1:-}"` test -n "${shunit_actual_}" assertTrue "${shunit_message_}" $? shunit_return=$? unset shunit_actual_ shunit_message_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_NOT_NULL_='eval assertNotNull --lineno "${LINENO:-}"' # Assert that two values are the same (i.e. equal to one another). # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertSame() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertSame() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi assertEquals "${shunit_message_}" "$1" "$2" shunit_return=$? unset shunit_message_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_SAME_='eval assertSame --lineno "${LINENO:-}"' # Assert that two values are not the same (i.e. not equal to one another). # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertNotSame() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "assertNotSame() requires two or three arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_:-}$1" shift fi assertNotEquals "${shunit_message_}" "$1" "$2" shunit_return=$? unset shunit_message_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_NOT_SAME_='eval assertNotSame --lineno "${LINENO:-}"' # Assert that a value or shell test condition is true. # # In shell, a value of 0 is true and a non-zero value is false. Any integer # value passed can thereby be tested. # # Shell supports much more complicated tests though, and a means to support # them was needed. As such, this function tests that conditions are true or # false through evaluation rather than just looking for a true or false. # # The following test will succeed: # assertTrue 0 # assertTrue "[ 34 -gt 23 ]" # The following test will fail with a message: # assertTrue 123 # assertTrue "test failed" "[ -r '/non/existent/file' ]" # # Args: # message: string: failure message [optional] # condition: string: integer value or shell conditional statement # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertTrue() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 1 -o $# -gt 2 ]; then _shunit_error "assertTrue() takes one or two arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_condition_=$1 # See if condition is an integer, i.e. a return value. shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` shunit_return=${SHUNIT_TRUE} if command [ -z "${shunit_condition_}" ]; then # Null condition. shunit_return=${SHUNIT_FALSE} elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] then # Possible return value. Treating 0 as true, and non-zero as false. command [ "${shunit_condition_}" -ne 0 ] && shunit_return=${SHUNIT_FALSE} else # Hopefully... a condition. ( eval "${shunit_condition_}" ) >/dev/null 2>&1 command [ $? -ne 0 ] && shunit_return=${SHUNIT_FALSE} fi # Record the test. if command [ ${shunit_return} -eq ${SHUNIT_TRUE} ]; then _shunit_assertPass else _shunit_assertFail "${shunit_message_}" fi unset shunit_message_ shunit_condition_ shunit_match_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _ASSERT_TRUE_='eval assertTrue --lineno "${LINENO:-}"' # Assert that a value or shell test condition is false. # # In shell, a value of 0 is true and a non-zero value is false. Any integer # value passed can thereby be tested. # # Shell supports much more complicated tests though, and a means to support # them was needed. As such, this function tests that conditions are true or # false through evaluation rather than just looking for a true or false. # # The following test will succeed: # assertFalse 1 # assertFalse "[ 'apples' = 'oranges' ]" # The following test will fail with a message: # assertFalse 0 # assertFalse "test failed" "[ 1 -eq 1 -a 2 -eq 2 ]" # # Args: # message: string: failure message [optional] # condition: string: integer value or shell conditional statement # Returns: # integer: success (TRUE/FALSE/ERROR constant) assertFalse() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 1 -o $# -gt 2 ]; then _shunit_error "assertFalse() requires one or two arguments; $# given" _shunit_assertFail return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_condition_=$1 # See if condition is an integer, i.e. a return value. shunit_match_=`expr "${shunit_condition_}" : '\([0-9]*\)'` shunit_return=${SHUNIT_TRUE} if command [ -z "${shunit_condition_}" ]; then # Null condition. shunit_return=${SHUNIT_FALSE} elif command [ -n "${shunit_match_}" -a "${shunit_condition_}" = "${shunit_match_}" ] then # Possible return value. Treating 0 as true, and non-zero as false. command [ "${shunit_condition_}" -eq 0 ] && shunit_return=${SHUNIT_FALSE} else # Hopefully... a condition. ( eval "${shunit_condition_}" ) >/dev/null 2>&1 command [ $? -eq 0 ] && shunit_return=${SHUNIT_FALSE} fi # Record the test. if command [ "${shunit_return}" -eq "${SHUNIT_TRUE}" ]; then _shunit_assertPass else _shunit_assertFail "${shunit_message_}" fi unset shunit_message_ shunit_condition_ shunit_match_ return "${shunit_return}" } # shellcheck disable=SC2016,SC2034 _ASSERT_FALSE_='eval assertFalse --lineno "${LINENO:-}"' #----------------------------------------------------------------------------- # Failure functions. # # Records a test failure. # # Args: # message: string: failure message [optional] # Returns: # integer: success (TRUE/FALSE/ERROR constant) fail() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -gt 1 ]; then _shunit_error "fail() requires zero or one arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 1 ]; then shunit_message_="${shunit_message_}$1" shift fi _shunit_assertFail "${shunit_message_}" unset shunit_message_ return ${SHUNIT_FALSE} } # shellcheck disable=SC2016,SC2034 _FAIL_='eval fail --lineno "${LINENO:-}"' # Records a test failure, stating two values were not equal. # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) failNotEquals() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "failNotEquals() requires one or two arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_expected_=$1 shunit_actual_=$2 shunit_message_=${shunit_message_%% } _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected:<${shunit_expected_}> but was:<${shunit_actual_}>" unset shunit_message_ shunit_expected_ shunit_actual_ return ${SHUNIT_FALSE} } # shellcheck disable=SC2016,SC2034 _FAIL_NOT_EQUALS_='eval failNotEquals --lineno "${LINENO:-}"' # Records a test failure, stating a value was found. # # Args: # message: string: failure message [optional] # content: string: found value # Returns: # integer: success (TRUE/FALSE/ERROR constant) failFound() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 1 -o $# -gt 2 ]; then _shunit_error "failFound() requires one or two arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_message_=${shunit_message_%% } _shunit_assertFail "${shunit_message_:+${shunit_message_} }Found" unset shunit_message_ return ${SHUNIT_FALSE} } # shellcheck disable=SC2016,SC2034 _FAIL_FOUND_='eval failFound --lineno "${LINENO:-}"' # Records a test failure, stating a content was not found. # # Args: # message: string: failure message [optional] # content: string: content not found # Returns: # integer: success (TRUE/FALSE/ERROR constant) failNotFound() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 1 -o $# -gt 2 ]; then _shunit_error "failNotFound() requires one or two arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 2 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_content_=$1 shunit_message_=${shunit_message_%% } _shunit_assertFail "${shunit_message_:+${shunit_message_} }Not found:<${shunit_content_}>" unset shunit_message_ shunit_content_ return ${SHUNIT_FALSE} } # shellcheck disable=SC2016,SC2034 _FAIL_NOT_FOUND_='eval failNotFound --lineno "${LINENO:-}"' # Records a test failure, stating two values should have been the same. # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) failSame() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "failSame() requires two or three arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi shunit_message_=${shunit_message_%% } _shunit_assertFail "${shunit_message_:+${shunit_message_} }expected not same" unset shunit_message_ return ${SHUNIT_FALSE} } # shellcheck disable=SC2016,SC2034 _FAIL_SAME_='eval failSame --lineno "${LINENO:-}"' # Records a test failure, stating two values were not equal. # # This is functionally equivalent to calling failNotEquals(). # # Args: # message: string: failure message [optional] # expected: string: expected value # actual: string: actual value # Returns: # integer: success (TRUE/FALSE/ERROR constant) failNotSame() { # shellcheck disable=SC2090 ${_SHUNIT_LINENO_} if command [ $# -lt 2 -o $# -gt 3 ]; then _shunit_error "failNotSame() requires one or two arguments; $# given" return ${SHUNIT_ERROR} fi _shunit_shouldSkip && return ${SHUNIT_TRUE} shunit_message_=${__shunit_lineno} if command [ $# -eq 3 ]; then shunit_message_="${shunit_message_}$1" shift fi failNotEquals "${shunit_message_}" "$1" "$2" shunit_return=$? unset shunit_message_ return ${shunit_return} } # shellcheck disable=SC2016,SC2034 _FAIL_NOT_SAME_='eval failNotSame --lineno "${LINENO:-}"' #----------------------------------------------------------------------------- # Skipping functions. # # Force remaining assert and fail functions to be "skipped". # # This function forces the remaining assert and fail functions to be "skipped", # i.e. they will have no effect. Each function skipped will be recorded so that # the total of asserts and fails will not be altered. # # Args: # None startSkipping() { __shunit_skip=${SHUNIT_TRUE}; } # Resume the normal recording behaviour of assert and fail calls. # # Args: # None endSkipping() { __shunit_skip=${SHUNIT_FALSE}; } # Returns the state of assert and fail call skipping. # # Args: # None # Returns: # boolean: (TRUE/FALSE constant) isSkipping() { return ${__shunit_skip}; } #----------------------------------------------------------------------------- # Suite functions. # # Stub. This function should contains all unit test calls to be made. # # DEPRECATED (as of 2.1.0) # # This function can be optionally overridden by the user in their test suite. # # If this function exists, it will be called when shunit2 is sourced. If it # does not exist, shunit2 will search the parent script for all functions # beginning with the word 'test', and they will be added dynamically to the # test suite. # # This function should be overridden by the user in their unit test suite. # Note: see _shunit_mktempFunc() for actual implementation # # Args: # None #suite() { :; } # DO NOT UNCOMMENT THIS FUNCTION # Adds a function name to the list of tests schedule for execution. # # This function should only be called from within the suite() function. # # Args: # function: string: name of a function to add to current unit test suite suite_addTest() { shunit_func_=${1:-} __shunit_suite="${__shunit_suite:+${__shunit_suite} }${shunit_func_}" __shunit_testsTotal=`expr ${__shunit_testsTotal} + 1` unset shunit_func_ } # Stub. This function will be called once before any tests are run. # # Common one-time environment preparation tasks shared by all tests can be # defined here. # # This function should be overridden by the user in their unit test suite. # Note: see _shunit_mktempFunc() for actual implementation # # Args: # None #oneTimeSetUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION # Stub. This function will be called once after all tests are finished. # # Common one-time environment cleanup tasks shared by all tests can be defined # here. # # This function should be overridden by the user in their unit test suite. # Note: see _shunit_mktempFunc() for actual implementation # # Args: # None #oneTimeTearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION # Stub. This function will be called before each test is run. # # Common environment preparation tasks shared by all tests can be defined here. # # This function should be overridden by the user in their unit test suite. # Note: see _shunit_mktempFunc() for actual implementation # # Args: # None #setUp() { :; } # DO NOT UNCOMMENT THIS FUNCTION # Note: see _shunit_mktempFunc() for actual implementation # Stub. This function will be called after each test is run. # # Common environment cleanup tasks shared by all tests can be defined here. # # This function should be overridden by the user in their unit test suite. # Note: see _shunit_mktempFunc() for actual implementation # # Args: # None #tearDown() { :; } # DO NOT UNCOMMENT THIS FUNCTION #------------------------------------------------------------------------------ # Internal shUnit2 functions. # # Create a temporary directory to store various run-time files in. # # This function is a cross-platform temporary directory creation tool. Not all # OSes have the `mktemp` function, so one is included here. # # Args: # None # Outputs: # string: the temporary directory that was created _shunit_mktempDir() { # Try the standard `mktemp` function. ( exec mktemp -dqt shunit.XXXXXX 2>/dev/null ) && return # The standard `mktemp` didn't work. Use our own. # shellcheck disable=SC2039 if command [ -r '/dev/urandom' -a -x '/usr/bin/od' ]; then _shunit_random_=`/usr/bin/od -vAn -N4 -tx4 "${_shunit_file_}" #! /bin/sh exit ${SHUNIT_TRUE} EOF command chmod +x "${_shunit_file_}" done unset _shunit_file_ } # Final cleanup function to leave things as we found them. # # Besides removing the temporary directory, this function is in charge of the # final exit code of the unit test. The exit code is based on how the script # was ended (e.g. normal exit, or via Ctrl-C). # # Args: # name: string: name of the trap called (specified when trap defined) _shunit_cleanup() { _shunit_name_=$1 case "${_shunit_name_}" in EXIT) ;; INT) _shunit_signal_=130 ;; # 2+128 TERM) _shunit_signal_=143 ;; # 15+128 *) _shunit_error "unrecognized trap value (${_shunit_name_})" _shunit_signal_=0 ;; esac if command [ "${_shunit_name_}" != 'EXIT' ]; then _shunit_warn "trapped and now handling the (${_shunit_name_}) signal" fi # Do our work. if command [ ${__shunit_clean} -eq ${SHUNIT_FALSE} ]; then # Ensure tear downs are only called once. __shunit_clean=${SHUNIT_TRUE} tearDown command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_warn "tearDown() returned non-zero return code." oneTimeTearDown command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_warn "oneTimeTearDown() returned non-zero return code." command rm -fr "${__shunit_tmpDir}" fi if command [ "${_shunit_name_}" != 'EXIT' ]; then # Handle all non-EXIT signals. trap - 0 # Disable EXIT trap. exit ${_shunit_signal_} elif command [ ${__shunit_reportGenerated} -eq ${SHUNIT_FALSE} ]; then _shunit_assertFail 'unknown failure encountered running a test' _shunit_generateReport exit ${SHUNIT_ERROR} fi unset _shunit_name_ _shunit_signal_ } # configureColor based on user color preference. # # Args: # color: string: color mode (one of `always`, `auto`, or `none`). _shunit_configureColor() { _shunit_color_=${SHUNIT_FALSE} # By default, no color. case $1 in 'always') _shunit_color_=${SHUNIT_TRUE} ;; 'auto') command [ "`_shunit_colors`" -ge 8 ] && _shunit_color_=${SHUNIT_TRUE} ;; 'none') ;; *) _shunit_fatal "unrecognized color option '$1'" ;; esac case ${_shunit_color_} in ${SHUNIT_TRUE}) __shunit_ansi_none=${__SHUNIT_ANSI_NONE} __shunit_ansi_red=${__SHUNIT_ANSI_RED} __shunit_ansi_green=${__SHUNIT_ANSI_GREEN} __shunit_ansi_yellow=${__SHUNIT_ANSI_YELLOW} __shunit_ansi_cyan=${__SHUNIT_ANSI_CYAN} ;; ${SHUNIT_FALSE}) __shunit_ansi_none='' __shunit_ansi_red='' __shunit_ansi_green='' __shunit_ansi_yellow='' __shunit_ansi_cyan='' ;; esac unset _shunit_color_ _shunit_tput_ } # colors returns the number of supported colors for the TERM. _shunit_colors() { _shunit_tput_=`${SHUNIT_CMD_TPUT} colors 2>/dev/null` if command [ $? -eq 0 ]; then echo "${_shunit_tput_}" else echo 16 fi unset _shunit_tput_ } # The actual running of the tests happens here. # # Args: # None _shunit_execSuite() { for _shunit_test_ in ${__shunit_suite}; do __shunit_testSuccess=${SHUNIT_TRUE} # Disable skipping. endSkipping # Execute the per-test setup function. setUp command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_fatal "setup() returned non-zero return code." # Execute the test. echo "${__SHUNIT_TEST_PREFIX}${_shunit_test_}" eval "${_shunit_test_}" if command [ $? -ne ${SHUNIT_TRUE} ]; then _shunit_error "${_shunit_test_}() returned non-zero return code." __shunit_testSuccess=${SHUNIT_ERROR} _shunit_incFailedCount fi # Execute the per-test tear-down function. tearDown command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_fatal "tearDown() returned non-zero return code." # Update stats. if command [ ${__shunit_testSuccess} -eq ${SHUNIT_TRUE} ]; then __shunit_testsPassed=`expr ${__shunit_testsPassed} + 1` else __shunit_testsFailed=`expr ${__shunit_testsFailed} + 1` fi done unset _shunit_test_ } # Generates the user friendly report with appropriate OK/FAILED message. # # Args: # None # Output: # string: the report of successful and failed tests, as well as totals. _shunit_generateReport() { command [ "${__shunit_reportGenerated}" -eq ${SHUNIT_TRUE} ] && return _shunit_ok_=${SHUNIT_TRUE} # If no exit code was provided, determine an appropriate one. command [ "${__shunit_testsFailed}" -gt 0 \ -o ${__shunit_testSuccess} -eq ${SHUNIT_FALSE} ] \ && _shunit_ok_=${SHUNIT_FALSE} echo _shunit_msg_="Ran ${__shunit_ansi_cyan}${__shunit_testsTotal}${__shunit_ansi_none}" if command [ "${__shunit_testsTotal}" -eq 1 ]; then ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} test." else ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_} tests." fi if command [ ${_shunit_ok_} -eq ${SHUNIT_TRUE} ]; then _shunit_msg_="${__shunit_ansi_green}OK${__shunit_ansi_none}" command [ "${__shunit_assertsSkipped}" -gt 0 ] \ && _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none})" else _shunit_msg_="${__shunit_ansi_red}FAILED${__shunit_ansi_none}" _shunit_msg_="${_shunit_msg_} (${__shunit_ansi_red}failures=${__shunit_assertsFailed}${__shunit_ansi_none}" command [ "${__shunit_assertsSkipped}" -gt 0 ] \ && _shunit_msg_="${_shunit_msg_},${__shunit_ansi_yellow}skipped=${__shunit_assertsSkipped}${__shunit_ansi_none}" _shunit_msg_="${_shunit_msg_})" fi echo ${__SHUNIT_CMD_ECHO_ESC} "${_shunit_msg_}" __shunit_reportGenerated=${SHUNIT_TRUE} unset _shunit_msg_ _shunit_ok_ } # Test for whether a function should be skipped. # # Args: # None # Returns: # boolean: whether the test should be skipped (TRUE/FALSE constant) _shunit_shouldSkip() { command [ ${__shunit_skip} -eq ${SHUNIT_FALSE} ] && return ${SHUNIT_FALSE} _shunit_assertSkip } # Records a successful test. # # Args: # None _shunit_assertPass() { __shunit_assertsPassed=`expr ${__shunit_assertsPassed} + 1` __shunit_assertsTotal=`expr ${__shunit_assertsTotal} + 1` } # Records a test failure. # # Args: # message: string: failure message to provide user _shunit_assertFail() { __shunit_testSuccess=${SHUNIT_FALSE} _shunit_incFailedCount \[ $# -gt 0 ] && ${__SHUNIT_CMD_ECHO_ESC} \ "${__shunit_ansi_red}ASSERT:${__shunit_ansi_none}$*" } # Increment the count of failed asserts. # # Args: # none _shunit_incFailedCount() { __shunit_assertsFailed=`expr "${__shunit_assertsFailed}" + 1` __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` } # Records a skipped test. # # Args: # None _shunit_assertSkip() { __shunit_assertsSkipped=`expr "${__shunit_assertsSkipped}" + 1` __shunit_assertsTotal=`expr "${__shunit_assertsTotal}" + 1` } # Prepare a script filename for sourcing. # # Args: # script: string: path to a script to source # Returns: # string: filename prefixed with ./ (if necessary) _shunit_prepForSourcing() { _shunit_script_=$1 case "${_shunit_script_}" in /*|./*) echo "${_shunit_script_}" ;; *) echo "./${_shunit_script_}" ;; esac unset _shunit_script_ } # Escape a character in a string. # # Args: # c: string: unescaped character # s: string: to escape character in # Returns: # string: with escaped character(s) _shunit_escapeCharInStr() { command [ -n "$2" ] || return # No point in doing work on an empty string. # Note: using shorter variable names to prevent conflicts with # _shunit_escapeCharactersInString(). _shunit_c_=$1 _shunit_s_=$2 # Escape the character. # shellcheck disable=SC1003,SC2086 echo ''${_shunit_s_}'' |command sed 's/\'${_shunit_c_}'/\\\'${_shunit_c_}'/g' unset _shunit_c_ _shunit_s_ } # Escape a character in a string. # # Args: # str: string: to escape characters in # Returns: # string: with escaped character(s) _shunit_escapeCharactersInString() { command [ -n "$1" ] || return # No point in doing work on an empty string. _shunit_str_=$1 # Note: using longer variable names to prevent conflicts with # _shunit_escapeCharInStr(). for _shunit_char_ in '"' '$' "'" '`'; do _shunit_str_=`_shunit_escapeCharInStr "${_shunit_char_}" "${_shunit_str_}"` done echo "${_shunit_str_}" unset _shunit_char_ _shunit_str_ } # Extract list of functions to run tests against. # # Args: # script: string: name of script to extract functions from # Returns: # string: of function names _shunit_extractTestFunctions() { _shunit_script_=$1 # Extract the lines with test function names, strip of anything besides the # function name, and output everything on a single line. _shunit_regex_='^\s*((function test[A-Za-z0-9_-]*)|(test[A-Za-z0-9_-]* *\(\)))' # shellcheck disable=SC2196 egrep "${_shunit_regex_}" "${_shunit_script_}" \ |command sed 's/^[^A-Za-z0-9_-]*//;s/^function //;s/\([A-Za-z0-9_-]*\).*/\1/g' \ |xargs unset _shunit_regex_ _shunit_script_ } #------------------------------------------------------------------------------ # Main. # # Determine the operating mode. if command [ $# -eq 0 -o "${1:-}" = '--' ]; then __shunit_script=${__SHUNIT_PARENT} __shunit_mode=${__SHUNIT_MODE_SOURCED} else __shunit_script=$1 command [ -r "${__shunit_script}" ] || \ _shunit_fatal "unable to read from ${__shunit_script}" __shunit_mode=${__SHUNIT_MODE_STANDALONE} fi # Create a temporary storage location. __shunit_tmpDir=`_shunit_mktempDir` # Provide a public temporary directory for unit test scripts. # TODO(kward): document this. SHUNIT_TMPDIR="${__shunit_tmpDir}/tmp" command mkdir "${SHUNIT_TMPDIR}" # Setup traps to clean up after ourselves. trap '_shunit_cleanup EXIT' 0 trap '_shunit_cleanup INT' 2 trap '_shunit_cleanup TERM' 15 # Create phantom functions to work around issues with Cygwin. _shunit_mktempFunc PATH="${__shunit_tmpDir}:${PATH}" # Make sure phantom functions are executable. This will bite if `/tmp` (or the # current `$TMPDIR`) points to a path on a partition that was mounted with the # 'noexec' option. The noexec command was created with `_shunit_mktempFunc()`. noexec 2>/dev/null || _shunit_fatal \ 'Please declare TMPDIR with path on partition with exec permission.' # We must manually source the tests in standalone mode. if command [ "${__shunit_mode}" = "${__SHUNIT_MODE_STANDALONE}" ]; then # shellcheck disable=SC1090 command . "`_shunit_prepForSourcing \"${__shunit_script}\"`" fi # Configure default output coloring behaviour. _shunit_configureColor "${SHUNIT_COLOR}" # Execute the oneTimeSetUp function (if it exists). oneTimeSetUp command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_fatal "oneTimeSetUp() returned non-zero return code." # Command line selected tests or suite selected tests if command [ "$#" -ge 2 ]; then # Argument $1 is either the filename of tests or '--'; either way, skip it. shift # Remaining arguments ($2 .. $#) are assumed to be test function names. # Iterate through all remaining args in "$@" in a POSIX (likely portable) way. # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop for _shunit_arg_ do suite_addTest "${_shunit_arg_}" done unset _shunit_arg_ else # Execute the suite function defined in the parent test script. # DEPRECATED as of 2.1.0. suite fi # If no tests or suite specified, dynamically build a list of functions. if command [ -z "${__shunit_suite}" ]; then shunit_funcs_=`_shunit_extractTestFunctions "${__shunit_script}"` for shunit_func_ in ${shunit_funcs_}; do suite_addTest "${shunit_func_}" done fi unset shunit_func_ shunit_funcs_ # Execute the suite of unit tests. _shunit_execSuite # Execute the oneTimeTearDown function (if it exists). oneTimeTearDown command [ $? -eq ${SHUNIT_TRUE} ] \ || _shunit_fatal "oneTimeTearDown() returned non-zero return code." # Generate a report summary. _shunit_generateReport # That's it folks. command [ "${__shunit_testsFailed}" -eq 0 ] exit $? ================================================ FILE: scripts/spelling.sh ================================================ #!/bin/bash npx cspell --no-progress "**/*.{sh,go,md}" ================================================ FILE: scripts/test-docker.sh ================================================ #! /bin/bash set -e docker build . -t temp docker run --rm -it --entrypoint sh temp -c 'touch a' ================================================ FILE: scripts/test.sh ================================================ #!/bin/bash go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test') ================================================ FILE: scripts/xcompile.sh ================================================ #!/bin/bash set -eo pipefail # You may need to go install github.com/goreleaser/goreleaser/v2@latest first GORELEASER="goreleaser build --clean" if [ -z "$CI" ]; then GORELEASER+=" --snapshot" fi mkdir -p build $GORELEASER cp yq.1 build/yq.1 cd build # Remove artifacts from goreleaser rm artifacts.json config.yaml metadata.json find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh tar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh rm yq_windows_386.exe.tar.gz rm yq_windows_amd64.exe.tar.gz rm yq_windows_arm64.exe.tar.gz zip yq_windows_386.zip yq_windows_386.exe zip yq_windows_amd64.zip yq_windows_amd64.exe zip yq_windows_arm64.zip yq_windows_arm64.exe rm yq.1 rhash -r -a . -o checksums rhash -r -a --bsd . -o checksums-bsd rhash --list-hashes > checksums_hashes_order cp ../scripts/extract-checksum.sh . ================================================ FILE: snap/snapcraft.yaml ================================================ name: yq version: 'v4.52.4' summary: A lightweight and portable command-line data file processor description: | `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. base: core24 grade: stable # devel|stable. must be 'stable' to release into candidate/stable channels confinement: strict platforms: amd64: build-on: [amd64] build-for: [amd64] arm64: build-on: [arm64] build-for: [arm64] armhf: build-on: [armhf] build-for: [armhf] s390x: build-on: [s390x] build-for: [s390x] ppc64el: build-on: [ppc64el] build-for: [ppc64el] apps: yq: command: bin/yq plugs: [home, removable-media] parts: yq: plugin: go build-environment: - CGO_ENABLED: 0 source: https://github.com/mikefarah/yq.git source-tag: v4.52.4 build-snaps: - go/latest/stable ================================================ FILE: test/format_test.go ================================================ package test import ( "testing" "github.com/mikefarah/yq/v4/pkg/yqlib" ) // only test format detection based on filename extension func TestFormatStringFromFilename(t *testing.T) { cases := []struct { filename string expected string }{ // filenames that have extensions {"file.yaml", "yaml"}, {"FILE.JSON", "json"}, {"file.properties", "properties"}, {"file.xml", "xml"}, {"file.unknown", "unknown"}, // filenames without extensions {"file", "yaml"}, {"a.dir/file", "yaml"}, {"file.", "yaml"}, {".", "yaml"}, {"", "yaml"}, } for _, c := range cases { result := yqlib.FormatStringFromFilename(c.filename) if result != c.expected { t.Errorf("FormatStringFromFilename(%q) = %q, wanted: %q", c.filename, result, c.expected) } } } ================================================ FILE: test/utils.go ================================================ package test import ( "bufio" "bytes" "fmt" "reflect" "testing" "github.com/pkg/diff" "github.com/pkg/diff/write" ) func printDifference(t *testing.T, expectedValue interface{}, actualValue interface{}) { opts := []write.Option{write.TerminalColor()} var differenceBuffer bytes.Buffer expectedString := fmt.Sprintf("%v", expectedValue) actualString := fmt.Sprintf("%v", actualValue) if err := diff.Text("expected", "actual", expectedString, actualString, bufio.NewWriter(&differenceBuffer), opts...); err != nil { t.Error(err) } else { t.Error(differenceBuffer.String()) } } func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) { t.Helper() if expectedValue != actualValue { printDifference(t, expectedValue, actualValue) } } func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) { t.Helper() if !reflect.DeepEqual(expectedValue, actualValue) { printDifference(t, expectedValue, actualValue) } } func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { t.Helper() if !reflect.DeepEqual(expectedValue, actualValue) { t.Error(context) printDifference(t, expectedValue, actualValue) } } func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { t.Helper() opts := []write.Option{write.TerminalColor()} if expectedValue != actualValue { t.Error(context) var differenceBuffer bytes.Buffer if err := diff.Text("expected", "actual", expectedValue, actualValue, bufio.NewWriter(&differenceBuffer), opts...); err != nil { t.Error(err) } else { t.Error(differenceBuffer.String()) } } } ================================================ FILE: test.yq ================================================ #!./yq .a.b ================================================ FILE: utf8.csv ================================================ id,first,last 1,john,smith 1,jane,smith ================================================ FILE: yq.go ================================================ package main import ( "os" command "github.com/mikefarah/yq/v4/cmd" ) func main() { cmd := command.New() args := os.Args[1:] _, _, err := cmd.Find(args) if err != nil && args[0] != "__complete" && args[0] != "__completeNoDesc" { // default command when nothing matches... newArgs := []string{"eval"} cmd.SetArgs(append(newArgs, os.Args[1:]...)) } if err := cmd.Execute(); err != nil { os.Exit(1) } } ================================================ FILE: yq_test.go ================================================ package main import ( "testing" command "github.com/mikefarah/yq/v4/cmd" ) func TestMainFunction(t *testing.T) { // This is a basic smoke test for the main function // We can't easily test the main function directly since it calls os.Exit // But we can test the logic that would be executed cmd := command.New() if cmd == nil { t.Fatal("command.New() returned nil") } if cmd.Use != "yq" { t.Errorf("Expected command Use to be 'yq', got %q", cmd.Use) } } func TestMainFunctionLogic(t *testing.T) { // Test the logic that would be executed in main() cmd := command.New() args := []string{} _, _, err := cmd.Find(args) if err != nil { t.Errorf("Expected no error with empty args, but got: %v", err) } args = []string{"invalid-command"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error when invalid command found, but got nil") } args = []string{"eval"} _, _, err = cmd.Find(args) if err != nil { t.Errorf("Expected no error with valid command 'eval', got: %v", err) } args = []string{"__complete"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error when no command found for '__complete', but got nil") } args = []string{"__completeNoDesc"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error when no command found for '__completeNoDesc', but got nil") } } func TestMainFunctionWithArgs(t *testing.T) { // Test the argument processing logic cmd := command.New() args := []string{} _, _, err := cmd.Find(args) if err != nil { t.Errorf("Expected no error with empty args, but got: %v", err) } // When Find fails and args[0] is not "__complete", main would set args to ["eval"] + original args // This is the logic: newArgs := []string{"eval"} // cmd.SetArgs(append(newArgs, os.Args[1:]...)) args = []string{"invalid"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error with invalid command") } args = []string{"__complete"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error with __complete command") } args = []string{"__completeNoDesc"} _, _, err = cmd.Find(args) if err == nil { t.Error("Expected error with __completeNoDesc command") } } func TestMainFunctionExecution(t *testing.T) { // Test that the command can be executed without crashing cmd := command.New() cmd.SetArgs([]string{"--version"}) // We can't easily test os.Exit(1) behaviour, but we can test that // the command structure is correct and can be configured if cmd == nil { t.Fatal("Command should not be nil") } if cmd.Use != "yq" { t.Errorf("Expected command Use to be 'yq', got %q", cmd.Use) } } func TestMainFunctionErrorHandling(t *testing.T) { // Test the error handling logic that would be in main() cmd := command.New() args := []string{"nonexistent-command"} _, _, err := cmd.Find(args) if err == nil { t.Error("Expected error with nonexistent command") } // The main function logic would be: // if err != nil && args[0] != "__complete" { // newArgs := []string{"eval"} // cmd.SetArgs(append(newArgs, os.Args[1:]...)) // } // Test that this logic would work if args[0] != "__complete" { // This is what main() would do newArgs := []string{"eval"} cmd.SetArgs(append(newArgs, args...)) // We can't easily verify the args were set correctly since cmd.Args is a function // But we can test that SetArgs doesn't crash and the command is still valid if cmd == nil { t.Error("Command should not be nil after SetArgs") } _, _, err := cmd.Find([]string{"eval"}) if err != nil { t.Errorf("Should be able to find eval command after SetArgs: %v", err) } } } func TestMainFunctionWithCompletionCommand(t *testing.T) { // Test that __complete command doesn't trigger default command logic cmd := command.New() args := []string{"__complete"} _, _, err := cmd.Find(args) if err == nil { t.Error("Expected error with __complete command") } // The main function logic would be: // if err != nil && args[0] != "__complete" { // // This should NOT execute for __complete // } // Verify that __complete doesn't trigger the default command logic if args[0] == "__complete" { // This means the default command logic should NOT execute t.Log("__complete command correctly identified, default command logic should not execute") } } func TestMainFunctionWithCompletionNoDescCommand(t *testing.T) { // Test that __complete command doesn't trigger default command logic cmd := command.New() args := []string{"__completeNoDesc"} _, _, err := cmd.Find(args) if err == nil { t.Error("Expected error with __completeNoDesc command") } // The main function logic would be: // if err != nil && args[0] != "__completeNoDesc" { // // This should NOT execute for __completeNoDesc // } // Verify that __completeNoDesc doesn't trigger the default command logic if args[0] == "__completeNoDesc" { // This means the default command logic should NOT execute t.Log("__completeNoDesc command correctly identified, default command logic should not execute") } } func TestMainFunctionIntegration(t *testing.T) { // Integration test to verify the main function logic works end-to-end cmd := command.New() cmd.SetArgs([]string{"eval", "--help"}) // This should not crash (we can't test the actual execution due to os.Exit) if cmd == nil { t.Fatal("Command should not be nil") } cmd2 := command.New() cmd2.SetArgs([]string{"invalid-command"}) // Simulate the main function logic args := []string{"invalid-command"} _, _, err := cmd2.Find(args) if err != nil { // This is what main() would do newArgs := []string{"eval"} cmd2.SetArgs(append(newArgs, args...)) } // We can't directly access cmd.Args since it's a function, but we can test // that SetArgs worked by ensuring the command is still functional if cmd2 == nil { t.Error("Command should not be nil after SetArgs") } _, _, err = cmd2.Find([]string{"eval"}) if err != nil { t.Errorf("Should be able to find eval command after SetArgs: %v", err) } }